diff --git a/.arclint b/.arclint index aab057084..4b07cf88c 100644 --- a/.arclint +++ b/.arclint @@ -1,247 +1,253 @@ { "linters": { "generated": { "type": "generated" }, "clang-format": { "type": "clang-format", "version": ">=8.0", "bin": ["clang-format-8", "clang-format"], "include": "(^src/.*\\.(h|c|cpp|mm)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "autopep8": { "type": "autopep8", "version": ">=1.3.4", "include": "(\\.py$)", "exclude": [ "(^contrib/gitian-builder/)" ], "flags": [ "--aggressive", "--ignore=W503,W504" ] }, "flake8": { "type": "flake8", "version": ">=3.0", "include": "(\\.py$)", "exclude": [ "(^contrib/gitian-builder/)" ], "flags": [ "--ignore=E501,E704,W503,W504" ] }, "lint-format-strings": { "type": "lint-format-strings", "include": "(^src/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "check-doc": { "type": "check-doc", "include": "(^src/.*\\.(h|c|cpp)$)" }, "lint-tests": { "type": "lint-tests", "include": "(^src/(seeder/|rpc/|wallet/)?test/.*\\.(cpp)$)" }, "lint-python-format": { "type": "lint-python-format", "include": "(\\.py$)", "exclude": [ "(^test/lint/lint-python-format\\.py$)", "(^contrib/gitian-builder/)" ] }, "phpcs": { "type": "phpcs", "include": "(\\.php$)", "exclude": [ "(^arcanist/__phutil_library_.+\\.php$)" ], "phpcs.standard": "arcanist/phpcs.xml" }, "lint-locale-dependence": { "type": "lint-locale-dependence", "include": "(^src/.*\\.(h|cpp)$)", "exclude": [ "(^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/))" ] }, "lint-cheader": { "type": "lint-cheader", "include": "(^src/.*\\.(h|cpp)$)", "exclude": [ "(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)" ] }, "spelling": { "type": "spelling", "exclude": [ "(^build-aux/m4/)", "(^depends/)", "(^doc/release-notes/)", "(^contrib/gitian-builder/)", "(^src/(qt/locale|secp256k1|univalue|leveldb)/)", "(^test/lint/dictionary/)" ], "spelling.dictionaries": [ "test/lint/dictionary/english.json" ] }, "lint-assert-with-side-effects": { "type": "lint-assert-with-side-effects", "include": "(^src/.*\\.(h|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "lint-include-quotes": { "type": "lint-include-quotes", "include": "(^src/.*\\.(h|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "lint-include-guard": { "type": "lint-include-guard", "include": "(^src/.*\\.h$)", "exclude": [ "(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)", "(^src/tinyformat.h$)" ] }, "lint-include-source": { "type": "lint-include-source", "include": "(^src/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "lint-stdint": { "type": "lint-stdint", "include": "(^src/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)", "(^src/compat/assumptions.h$)" ] }, "lint-source-filename": { "type": "lint-source-filename", "include": "(^src/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "lint-boost-dependencies": { "type": "lint-boost-dependencies", "include": "(^src/.*\\.(h|cpp)$)" }, "check-rpc-mappings": { "type": "check-rpc-mappings", "include": "(^src/(rpc/|wallet/rpc).*\\.cpp$)" }, "lint-python-encoding": { "type": "lint-python-encoding", "include": "(\\.py$)", "exclude": [ "(^contrib/gitian-builder/)" ] }, "lint-python-shebang": { "type": "lint-python-shebang", "include": "(\\.py$)", "exclude": [ "(__init__\\.py$)", "(^contrib/gitian-builder/)" ] }, "lint-bash-shebang": { "type": "lint-bash-shebang", "include": "(\\.sh$)", "exclude": [ "(^contrib/gitian-builder/)" ] }, "shellcheck": { "type": "shellcheck", "version": ">=0.7.0", "flags": [ "--external-sources", "--source-path=SCRIPTDIR" ], "include": "(\\.sh$)", "exclude": [ "(^contrib/gitian-builder/)", "(^src/(secp256k1|univalue)/)" ] }, "lint-shell-locale": { "type": "lint-shell-locale", "include": "(\\.sh$)", "exclude": [ "(^contrib/gitian-builder/)", "(^src/(secp256k1|univalue)/)", "(^cmake/utils/test_wrapper.sh)" ] }, "lint-cpp-void-parameters": { "type": "lint-cpp-void-parameters", "include": "(^src/.*\\.(h|cpp)$)", "exclude": [ "(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)", "(^src/compat/glibc_compat.cpp$)" ] }, "lint-logs": { "type": "lint-logs", "include": "(^src/.*\\.(h|cpp)$)" }, "lint-qt": { "type": "lint-qt", "include": "(^src/qt/.*\\.(h|cpp)$)", "exclude": [ "(^src/qt/(locale|forms|res)/)" ] }, "lint-doxygen": { "type": "lint-doxygen", "include": "(^src/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)" ] }, "lint-whitespace": { "type": "lint-whitespace", "include": "(\\.(ac|am|cmake|conf|in|include|json|m4|md|openrc|php|pl|sh|txt|yml)$)", "exclude": [ "(^contrib/gitian-builder/)", "(^src/(secp256k1|univalue|leveldb)/)" ] }, "lint-cppcheck": { "type": "lint-cppcheck", "include": "(^src/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)" ] }, "yamllint": { "type": "yamllint", "include": "(\\.(yml|yaml)$)" }, "lint-check-nonfatal": { "type": "lint-check-nonfatal", "include": [ "(^src/rpc/.*\\.(h|c|cpp)$)", "(^src/wallet/rpc*.*\\.(h|c|cpp)$)" ], "exclude": "(^src/rpc/server.cpp)" + }, + "lint-markdown": { + "type": "lint-markdown", + "include": [ + "(\\.md$)" + ] } } } diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 02b0f92be..1ffc6433a 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,37 +1,37 @@ This issue tracker is only for technical issues related to bitcoin-abc. General bitcoin questions and/or support requests and are best directed to the [Bitcoin StackExchange](https://bitcoin.stackexchange.com). -For reporting security issues, please see [DISCLOSURE_POLICY](DISCLOSURE_POLICY.md). +For reporting security issues, please see [DISCLOSURE_POLICY](/DISCLOSURE_POLICY.md). ### Describe the issue ### Can you reliably reproduce the issue? #### If so, please list the steps to reproduce below: 1. 2. 3. ### Expected behaviour Tell us what should happen ### Actual behaviour Tell us what happens instead ### Screenshots. If the issue is related to the GUI, screenshots can be added to this issue via drag & drop. ### What version of bitcoin-abc are you using? List the version number/commit ID, and if it is an official binary, self compiled or a distribution package such as PPA. ### Machine specs: - OS: - CPU: - RAM: - Disk size: - Disk Type (HD/SDD): ### Any extra information that might be useful in the debugging process. This is normally the contents of a `debug.log` or `config.log` file. Raw text or a link to a pastebin type site are preferred. diff --git a/arcanist/.phutil_module_cache b/arcanist/.phutil_module_cache index 9eaf87ac4..f76b9eaee 100644 --- a/arcanist/.phutil_module_cache +++ b/arcanist/.phutil_module_cache @@ -1 +1 @@ -{"__symbol_cache_version__":11,"ae5b7d3d6b8cf9598ce4abaf0cd56b21":{"have":{"class":{"ExtendedConfigurationDrivenLintEngine":19}},"need":{"function":{"newv":159,"pht":933},"class":{"ArcanistLintEngine":65,"ArcanistConfigurationDrivenLintEngine":171,"PhutilConsole":866},"class\/interface":{"ILintOnce":634}},"xmap":{"ExtendedConfigurationDrivenLintEngine":["ArcanistLintEngine"]}},"9780c82e3cf1cde0c46662563f00444f":{"have":{"class":{"ArcanistBitcoinABCConfiguration":13}},"need":{"function":{"pht":242},"class":{"ArcanistConfiguration":53,"ArcanistLintWorkflow":1652},"class\/interface":{"ArcanistWorkflow":385,"ArcanistNoEffectException":1371}},"xmap":{"ArcanistBitcoinABCConfiguration":["ArcanistConfiguration"]}},"7357e5ebbf0ab68606496d64ce336b2d":{"have":{"class":{"ArcanistLandBotWorkflow":91}},"need":{"function":{"phutil_console_format":265,"pht":1577,"execx":2181},"class":{"ArcanistWorkflow":123,"ArcanistUsageException":2251}},"xmap":{"ArcanistLandBotWorkflow":["ArcanistWorkflow"]}},"9285ad9415f8ebe564f7119e5a72c559":{"have":{"class":{"FormatStringLinter":146}},"need":{"function":{"pht":377,"csprintf":1492,"id":1872},"class":{"ArcanistExternalLinter":173,"ArcanistLintMessage":1879,"Filesystem":827,"ArcanistLinter":1956,"ArcanistLintSeverity":2044}},"xmap":{"FormatStringLinter":["ArcanistExternalLinter"]}},"2cbb6e7228d81557f777ad648704f343":{"have":{"class":{"IncludeQuotesLinter":100}},"need":{"function":{"pht":306},"class":{"ArcanistLinter":128,"ArcanistLintSeverity":663,"Filesystem":964}},"xmap":{"IncludeQuotesLinter":["ArcanistLinter"]}},"377eb7a4b08b962d2987624c96b99e5d":{"have":{"class":{"LocaleDependenceLinter":160}},"need":{"function":{"pht":5445},"class":{"ArcanistLinter":191,"ArcanistLintSeverity":5948,"Filesystem":6194}},"xmap":{"LocaleDependenceLinter":["ArcanistLinter"]}},"39694cf756ce58681ea84a028f3e007f":{"have":{"class":{"IncludeGuardLinter":98}},"need":{"function":{"pht":443},"class":{"ArcanistLinter":125,"ArcanistLintSeverity":796,"Filesystem":1162}},"xmap":{"IncludeGuardLinter":["ArcanistLinter"]}},"85936cbfc0decd1aae05001717b359da":{"have":{"class":{"LogLinter":116}},"need":{"function":{"pht":297},"class":{"ArcanistLinter":134,"ArcanistLintSeverity":580,"Filesystem":875}},"xmap":{"LogLinter":["ArcanistLinter"]}},"9c6223f7b59509a61f43d02842cc4bbc":{"have":{"class":{"CppVoidParameterLinter":119}},"need":{"function":{"pht":333},"class":{"ArcanistLinter":150,"ArcanistLintSeverity":648,"Filesystem":895}},"xmap":{"CppVoidParameterLinter":["ArcanistLinter"]}},"6f2f22dd0f259fb2eaa284b4fab3bc29":{"have":{"class":{"PythonFormatLinter":123}},"need":{"function":{"pht":353,"id":1838},"class":{"ArcanistExternalLinter":150,"ArcanistLintMessage":1845,"Filesystem":776,"ArcanistLinter":1970,"ArcanistLintSeverity":2053}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"454160ed72e88797129d84ee589de304":{"have":{"class":{"PythonShebangLinter":87}},"need":{"function":{"pht":290},"class":{"ArcanistLinter":115,"ArcanistLintSeverity":614,"Filesystem":854}},"xmap":{"PythonShebangLinter":["ArcanistLinter"]}},"38f0c676bff5192a344464142caaa253":{"have":{"class":{"CHeaderLinter":99}},"need":{"function":{"pht":611},"class":{"ArcanistLinter":121,"ArcanistLintSeverity":1060,"Filesystem":1307}},"xmap":{"CHeaderLinter":["ArcanistLinter"]}},"146347e1ef63e514a6634ada8de5b5c9":{"have":{"class":{"Qt5Linter":70}},"need":{"function":{"pht":258},"class":{"ArcanistLinter":88,"ArcanistLintSeverity":609,"Filesystem":836}},"xmap":{"Qt5Linter":["ArcanistLinter"]}},"26838a30f2792da86e8de3acf18a0ea2":{"have":{"class":{"CppCheckLinter":92}},"need":{"function":{"pht":6052,"execx":6412},"class":{"ArcanistExternalLinter":115,"ArcanistLintMessage":8880,"Filesystem":8002,"ArcanistLintSeverity":9189}},"xmap":{"CppCheckLinter":["ArcanistExternalLinter"]}},"09a933fbbf135320585be52750d93831":{"have":{"class":{"StdintLinter":90}},"need":{"function":{"pht":280},"class":{"ArcanistLinter":111,"ArcanistLintSeverity":589,"Filesystem":897}},"xmap":{"StdintLinter":["ArcanistLinter"]}},"729e2f379c278258d8040e156ba9da29":{"have":{"class":{"ShellLocaleLinter":107}},"need":{"function":{"pht":382},"class":{"ArcanistLinter":133,"ArcanistLintSeverity":734,"Filesystem":1044}},"xmap":{"ShellLocaleLinter":["ArcanistLinter"]}},"5ea58c19df0397ed8ee0f463d90d6c72":{"have":{"class":{"BoostDependenciesLinter":145}},"need":{"function":{"pht":330,"id":1440},"class":{"GlobalExternalLinter":177,"ArcanistLintMessage":1447,"Filesystem":609,"ArcanistLinter":1524,"ArcanistLintSeverity":1624}},"xmap":{"BoostDependenciesLinter":["GlobalExternalLinter"]}},"4c4adcac26dddfa5e350b85a40097755":{"have":{"class":{"BashShebangLinter":82}},"need":{"function":{"pht":281},"class":{"ArcanistLinter":108,"ArcanistLintSeverity":597,"Filesystem":837}},"xmap":{"BashShebangLinter":["ArcanistLinter"]}},"c9e595b23560664866ed4d6f2cbdb117":{"have":{"class":{"WhitespaceLinter":59}},"need":{"function":{"pht":258},"class":{"ArcanistLinter":84,"ArcanistLintSeverity":545,"Filesystem":791}},"xmap":{"WhitespaceLinter":["ArcanistLinter"]}},"e4678dca551a849892aff414d3c4e24e":{"have":{"class":{"DoxygenLinter":91}},"need":{"function":{"pht":382},"class":{"ArcanistLinter":113,"ArcanistLintSeverity":695,"Filesystem":943}},"xmap":{"DoxygenLinter":["ArcanistLinter"]}},"3cd6909c95569e2564deabbd1ec38bbb":{"have":{"class":{"FileNameLinter":103}},"need":{"function":{"pht":382},"class":{"ArcanistLinter":126,"ArcanistLintSeverity":737,"Filesystem":1160}},"xmap":{"FileNameLinter":["ArcanistLinter"]}},"2e11dd9ad67e594f863bc46ac59ea37e":{"have":{"class":{"GlobalExternalLinter":199}},"need":{"class":{"ArcanistExternalLinter":228},"interface":{"ILintOnce":262}},"xmap":{"GlobalExternalLinter":["ArcanistExternalLinter","ILintOnce"]}},"2809b09d2021203b43c57da33d1fe8bf":{"have":{"class":{"AssertWithSideEffectsLinter":210}},"need":{"function":{"pht":439},"class":{"ArcanistLinter":246,"ArcanistLintSeverity":926,"Filesystem":1170}},"xmap":{"AssertWithSideEffectsLinter":["ArcanistLinter"]}},"b2403124ec3e8be6cb4d10bf0f6c4134":{"have":{"interface":{"ILintOnce":69}},"need":[],"xmap":[]},"90a8b110dc475955f15bb81d37268cb5":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"execx":769,"id":1903},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1910,"Filesystem":1754,"ArcanistLinter":2017,"ArcanistLintSeverity":2095}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"9bb48ec0fe2e9ced8e27d42540d0571c":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":321,"id":1845},"class":{"GlobalExternalLinter":129,"ArcanistLintMessage":1852,"Filesystem":729,"ArcanistLinter":1900,"ArcanistLintSeverity":1986}},"xmap":{"CheckDocLinter":["GlobalExternalLinter"]}},"fbb3938ab9b412d7505335d9f7ed183d":{"have":{"class":{"ShellCheckLinter":95}},"need":{"function":{"pht":921,"execx":1443,"id":2414},"class":{"ArcanistExternalLinter":120,"ArcanistLintMessage":2421,"ArcanistLintSeverity":646,"ArcanistLinter":2469}},"xmap":{"ShellCheckLinter":["ArcanistExternalLinter"]}},"63d19a8745cb2e1200cc26488dc7ad25":{"have":{"class":{"CheckRpcMappingsLinter":131}},"need":{"function":{"pht":310,"id":1386},"class":{"GlobalExternalLinter":162,"ArcanistLintMessage":1393,"Filesystem":573,"ArcanistLinter":1443,"ArcanistLintSeverity":1544}},"xmap":{"CheckRpcMappingsLinter":["GlobalExternalLinter"]}},"24f5f28d739120159e375f587d3a68e4":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":781,"id":2235},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":2242,"Filesystem":2086,"ArcanistLinter":2349,"ArcanistLintSeverity":2427}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}},"e6c7c36001803d09d667e83285bd04e7":{"have":{"class":{"YamllintLinter":77}},"need":{"function":{"pht":399,"execx":907,"id":1711},"class":{"ArcanistExternalLinter":100,"ArcanistLintMessage":1718,"ArcanistLintSeverity":184,"ArcanistLinter":1766}},"xmap":{"YamllintLinter":["ArcanistExternalLinter"]}},"2f99c841a569f2029d13ea030dee007c":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2638},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2645,"Filesystem":776,"ArcanistLinter":2693,"ArcanistLintSeverity":2801}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}},"b4005bc083930dd85220bffb4c5f8bdd":{"have":{"class":{"PythonFileEncodingLinter":111}},"need":{"function":{"pht":321},"class":{"ArcanistLinter":144,"ArcanistLintSeverity":685,"Filesystem":990}},"xmap":{"PythonFileEncodingLinter":["ArcanistLinter"]}},"f151089cf79fdb8257b2272ed4782d88":{"have":{"class":{"IncludeSourceLinter":99}},"need":{"function":{"pht":391},"class":{"ArcanistLinter":127,"ArcanistLintSeverity":699,"Filesystem":938}},"xmap":{"IncludeSourceLinter":["ArcanistLinter"]}},"0dfd2c03e84436342a043927cde96305":{"have":{"class":{"CheckNonFatalOverAssertInRpc":268}},"need":{"function":{"pht":582},"class":{"ArcanistLinter":305,"ArcanistLintSeverity":863,"Filesystem":1144}},"xmap":{"CheckNonFatalOverAssertInRpc":["ArcanistLinter"]}}} \ No newline at end of file +{"__symbol_cache_version__":11,"7357e5ebbf0ab68606496d64ce336b2d":{"have":{"class":{"ArcanistLandBotWorkflow":91}},"need":{"function":{"phutil_console_format":265,"pht":1577,"execx":2181},"class":{"ArcanistWorkflow":123,"ArcanistUsageException":2251}},"xmap":{"ArcanistLandBotWorkflow":["ArcanistWorkflow"]}},"ae5b7d3d6b8cf9598ce4abaf0cd56b21":{"have":{"class":{"ExtendedConfigurationDrivenLintEngine":19}},"need":{"function":{"newv":159,"pht":933},"class":{"ArcanistLintEngine":65,"ArcanistConfigurationDrivenLintEngine":171,"PhutilConsole":866},"class\/interface":{"ILintOnce":634}},"xmap":{"ExtendedConfigurationDrivenLintEngine":["ArcanistLintEngine"]}},"90a8b110dc475955f15bb81d37268cb5":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"execx":769,"id":1903},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1910,"Filesystem":1754,"ArcanistLinter":2017,"ArcanistLintSeverity":2095}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"5ea58c19df0397ed8ee0f463d90d6c72":{"have":{"class":{"BoostDependenciesLinter":145}},"need":{"function":{"pht":330,"id":1440},"class":{"GlobalExternalLinter":177,"ArcanistLintMessage":1447,"Filesystem":609,"ArcanistLinter":1524,"ArcanistLintSeverity":1624}},"xmap":{"BoostDependenciesLinter":["GlobalExternalLinter"]}},"c9e595b23560664866ed4d6f2cbdb117":{"have":{"class":{"WhitespaceLinter":59}},"need":{"function":{"pht":258},"class":{"ArcanistLinter":84,"ArcanistLintSeverity":545,"Filesystem":791}},"xmap":{"WhitespaceLinter":["ArcanistLinter"]}},"6f2f22dd0f259fb2eaa284b4fab3bc29":{"have":{"class":{"PythonFormatLinter":123}},"need":{"function":{"pht":353,"id":1838},"class":{"ArcanistExternalLinter":150,"ArcanistLintMessage":1845,"Filesystem":776,"ArcanistLinter":1970,"ArcanistLintSeverity":2053}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"e4678dca551a849892aff414d3c4e24e":{"have":{"class":{"DoxygenLinter":91}},"need":{"function":{"pht":382},"class":{"ArcanistLinter":113,"ArcanistLintSeverity":695,"Filesystem":943}},"xmap":{"DoxygenLinter":["ArcanistLinter"]}},"4c4adcac26dddfa5e350b85a40097755":{"have":{"class":{"BashShebangLinter":82}},"need":{"function":{"pht":281},"class":{"ArcanistLinter":108,"ArcanistLintSeverity":597,"Filesystem":837}},"xmap":{"BashShebangLinter":["ArcanistLinter"]}},"f151089cf79fdb8257b2272ed4782d88":{"have":{"class":{"IncludeSourceLinter":99}},"need":{"function":{"pht":391},"class":{"ArcanistLinter":127,"ArcanistLintSeverity":699,"Filesystem":938}},"xmap":{"IncludeSourceLinter":["ArcanistLinter"]}},"b2403124ec3e8be6cb4d10bf0f6c4134":{"have":{"interface":{"ILintOnce":69}},"need":[],"xmap":[]},"63d19a8745cb2e1200cc26488dc7ad25":{"have":{"class":{"CheckRpcMappingsLinter":131}},"need":{"function":{"pht":310,"id":1386},"class":{"GlobalExternalLinter":162,"ArcanistLintMessage":1393,"Filesystem":573,"ArcanistLinter":1443,"ArcanistLintSeverity":1544}},"xmap":{"CheckRpcMappingsLinter":["GlobalExternalLinter"]}},"9285ad9415f8ebe564f7119e5a72c559":{"have":{"class":{"FormatStringLinter":146}},"need":{"function":{"pht":377,"csprintf":1492,"id":1872},"class":{"ArcanistExternalLinter":173,"ArcanistLintMessage":1879,"Filesystem":827,"ArcanistLinter":1956,"ArcanistLintSeverity":2044}},"xmap":{"FormatStringLinter":["ArcanistExternalLinter"]}},"0dfd2c03e84436342a043927cde96305":{"have":{"class":{"CheckNonFatalOverAssertInRpc":268}},"need":{"function":{"pht":582},"class":{"ArcanistLinter":305,"ArcanistLintSeverity":863,"Filesystem":1144}},"xmap":{"CheckNonFatalOverAssertInRpc":["ArcanistLinter"]}},"85936cbfc0decd1aae05001717b359da":{"have":{"class":{"LogLinter":116}},"need":{"function":{"pht":297},"class":{"ArcanistLinter":134,"ArcanistLintSeverity":580,"Filesystem":875}},"xmap":{"LogLinter":["ArcanistLinter"]}},"26838a30f2792da86e8de3acf18a0ea2":{"have":{"class":{"CppCheckLinter":92}},"need":{"function":{"pht":6052,"execx":6412},"class":{"ArcanistExternalLinter":115,"ArcanistLintMessage":8880,"Filesystem":8002,"ArcanistLintSeverity":9189}},"xmap":{"CppCheckLinter":["ArcanistExternalLinter"]}},"729e2f379c278258d8040e156ba9da29":{"have":{"class":{"ShellLocaleLinter":107}},"need":{"function":{"pht":382},"class":{"ArcanistLinter":133,"ArcanistLintSeverity":734,"Filesystem":1044}},"xmap":{"ShellLocaleLinter":["ArcanistLinter"]}},"09a933fbbf135320585be52750d93831":{"have":{"class":{"StdintLinter":90}},"need":{"function":{"pht":280},"class":{"ArcanistLinter":111,"ArcanistLintSeverity":589,"Filesystem":897}},"xmap":{"StdintLinter":["ArcanistLinter"]}},"2cbb6e7228d81557f777ad648704f343":{"have":{"class":{"IncludeQuotesLinter":100}},"need":{"function":{"pht":306},"class":{"ArcanistLinter":128,"ArcanistLintSeverity":663,"Filesystem":964}},"xmap":{"IncludeQuotesLinter":["ArcanistLinter"]}},"38f0c676bff5192a344464142caaa253":{"have":{"class":{"CHeaderLinter":99}},"need":{"function":{"pht":611},"class":{"ArcanistLinter":121,"ArcanistLintSeverity":1060,"Filesystem":1307}},"xmap":{"CHeaderLinter":["ArcanistLinter"]}},"3cd6909c95569e2564deabbd1ec38bbb":{"have":{"class":{"FileNameLinter":103}},"need":{"function":{"pht":382},"class":{"ArcanistLinter":126,"ArcanistLintSeverity":737,"Filesystem":1160}},"xmap":{"FileNameLinter":["ArcanistLinter"]}},"2e11dd9ad67e594f863bc46ac59ea37e":{"have":{"class":{"GlobalExternalLinter":199}},"need":{"class":{"ArcanistExternalLinter":228},"interface":{"ILintOnce":262}},"xmap":{"GlobalExternalLinter":["ArcanistExternalLinter","ILintOnce"]}},"146347e1ef63e514a6634ada8de5b5c9":{"have":{"class":{"Qt5Linter":70}},"need":{"function":{"pht":258},"class":{"ArcanistLinter":88,"ArcanistLintSeverity":609,"Filesystem":836}},"xmap":{"Qt5Linter":["ArcanistLinter"]}},"2f99c841a569f2029d13ea030dee007c":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2638},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2645,"Filesystem":776,"ArcanistLinter":2693,"ArcanistLintSeverity":2801}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}},"24f5f28d739120159e375f587d3a68e4":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":781,"id":2235},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":2242,"Filesystem":2086,"ArcanistLinter":2349,"ArcanistLintSeverity":2427}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}},"fbb3938ab9b412d7505335d9f7ed183d":{"have":{"class":{"ShellCheckLinter":95}},"need":{"function":{"pht":921,"execx":1443,"id":2414},"class":{"ArcanistExternalLinter":120,"ArcanistLintMessage":2421,"ArcanistLintSeverity":646,"ArcanistLinter":2469}},"xmap":{"ShellCheckLinter":["ArcanistExternalLinter"]}},"e6c7c36001803d09d667e83285bd04e7":{"have":{"class":{"YamllintLinter":77}},"need":{"function":{"pht":399,"execx":907,"id":1711},"class":{"ArcanistExternalLinter":100,"ArcanistLintMessage":1718,"ArcanistLintSeverity":184,"ArcanistLinter":1766}},"xmap":{"YamllintLinter":["ArcanistExternalLinter"]}},"39694cf756ce58681ea84a028f3e007f":{"have":{"class":{"IncludeGuardLinter":98}},"need":{"function":{"pht":443},"class":{"ArcanistLinter":125,"ArcanistLintSeverity":796,"Filesystem":1162}},"xmap":{"IncludeGuardLinter":["ArcanistLinter"]}},"9c6223f7b59509a61f43d02842cc4bbc":{"have":{"class":{"CppVoidParameterLinter":119}},"need":{"function":{"pht":333},"class":{"ArcanistLinter":150,"ArcanistLintSeverity":648,"Filesystem":895}},"xmap":{"CppVoidParameterLinter":["ArcanistLinter"]}},"454160ed72e88797129d84ee589de304":{"have":{"class":{"PythonShebangLinter":87}},"need":{"function":{"pht":290},"class":{"ArcanistLinter":115,"ArcanistLintSeverity":614,"Filesystem":854}},"xmap":{"PythonShebangLinter":["ArcanistLinter"]}},"9bb48ec0fe2e9ced8e27d42540d0571c":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":321,"id":1845},"class":{"GlobalExternalLinter":129,"ArcanistLintMessage":1852,"Filesystem":729,"ArcanistLinter":1900,"ArcanistLintSeverity":1986}},"xmap":{"CheckDocLinter":["GlobalExternalLinter"]}},"b4005bc083930dd85220bffb4c5f8bdd":{"have":{"class":{"PythonFileEncodingLinter":111}},"need":{"function":{"pht":321},"class":{"ArcanistLinter":144,"ArcanistLintSeverity":685,"Filesystem":990}},"xmap":{"PythonFileEncodingLinter":["ArcanistLinter"]}},"377eb7a4b08b962d2987624c96b99e5d":{"have":{"class":{"LocaleDependenceLinter":160}},"need":{"function":{"pht":5445},"class":{"ArcanistLinter":191,"ArcanistLintSeverity":5948,"Filesystem":6194}},"xmap":{"LocaleDependenceLinter":["ArcanistLinter"]}},"2809b09d2021203b43c57da33d1fe8bf":{"have":{"class":{"AssertWithSideEffectsLinter":210}},"need":{"function":{"pht":439},"class":{"ArcanistLinter":246,"ArcanistLintSeverity":926,"Filesystem":1170}},"xmap":{"AssertWithSideEffectsLinter":["ArcanistLinter"]}},"9780c82e3cf1cde0c46662563f00444f":{"have":{"class":{"ArcanistBitcoinABCConfiguration":13}},"need":{"function":{"pht":242},"class":{"ArcanistConfiguration":53,"ArcanistLintWorkflow":1652},"class\/interface":{"ArcanistWorkflow":385,"ArcanistNoEffectException":1371}},"xmap":{"ArcanistBitcoinABCConfiguration":["ArcanistConfiguration"]}},"5eecb9a1ee2fe5e65cd50ae6bff09ad5":{"have":{"function":{"startsWith":16},"class":{"MarkdownLinter":171}},"need":{"function":{"pht":361},"class":{"ArcanistLinter":194,"ArcanistLintSeverity":652,"Filesystem":1503}},"xmap":{"MarkdownLinter":["ArcanistLinter"]}}} \ No newline at end of file diff --git a/arcanist/__phutil_library_map__.php b/arcanist/__phutil_library_map__.php index eb44b00e5..12fd3d7b8 100644 --- a/arcanist/__phutil_library_map__.php +++ b/arcanist/__phutil_library_map__.php @@ -1,86 +1,90 @@ 2, 'class' => array( 'ArcanistBitcoinABCConfiguration' => 'configuration/ArcanistBitcoinABCConfiguration.php', 'ArcanistLandBotWorkflow' => 'workflow/ArcanistLandBotWorkflow.php', 'AssertWithSideEffectsLinter' => 'linter/AssertWithSideEffectsLinter.php', 'AutoPEP8FormatLinter' => 'linter/AutoPEP8Linter.php', 'BashShebangLinter' => 'linter/BashShebangLinter.php', 'BoostDependenciesLinter' => 'linter/BoostDependenciesLinter.php', 'CHeaderLinter' => 'linter/CHeaderLinter.php', 'CheckDocLinter' => 'linter/CheckDocLinter.php', 'CheckNonFatalOverAssertInRpc' => 'linter/CheckNonFatalOverAssertInRpc.php', 'CheckRpcMappingsLinter' => 'linter/CheckRpcMappingsLinter.php', 'ClangFormatLinter' => 'linter/ClangFormatLinter.php', 'CppCheckLinter' => 'linter/CppCheckLinter.php', 'CppVoidParameterLinter' => 'linter/CppVoidParameterLinter.php', 'DoxygenLinter' => 'linter/DoxygenLinter.php', 'ExtendedConfigurationDrivenLintEngine' => 'engine/ExtendedConfigurationDrivenLintEngine.php', 'FileNameLinter' => 'linter/FileNameLinter.php', 'FormatStringLinter' => 'linter/FormatStringLinter.php', 'GlobalExternalLinter' => 'linter/GlobalExternalLinter.php', 'ILintOnce' => 'linter/ILintOnce.php', 'IncludeGuardLinter' => 'linter/IncludeGuardLinter.php', 'IncludeQuotesLinter' => 'linter/IncludeQuotesLinter.php', 'IncludeSourceLinter' => 'linter/IncludeSourceLinter.php', 'LocaleDependenceLinter' => 'linter/LocaleDependenceLinter.php', 'LogLinter' => 'linter/LogLinter.php', + 'MarkdownLinter' => 'linter/MarkdownLinter.php', 'PythonFileEncodingLinter' => 'linter/PythonFileEncodingLinter.php', 'PythonFormatLinter' => 'linter/PythonFormatLinter.php', 'PythonShebangLinter' => 'linter/PythonShebangLinter.php', 'Qt5Linter' => 'linter/Qt5Linter.php', 'ShellCheckLinter' => 'linter/ShellCheckLinter.php', 'ShellLocaleLinter' => 'linter/ShellLocaleLinter.php', 'StdintLinter' => 'linter/StdintLinter.php', 'TestsLinter' => 'linter/TestsLinter.php', 'WhitespaceLinter' => 'linter/WhitespaceLinter.php', 'YamllintLinter' => 'linter/YamllintLinter.php', ), - 'function' => array(), + 'function' => array( + 'startsWith' => 'linter/MarkdownLinter.php', + ), 'xmap' => array( 'ArcanistBitcoinABCConfiguration' => 'ArcanistConfiguration', 'ArcanistLandBotWorkflow' => 'ArcanistWorkflow', 'AssertWithSideEffectsLinter' => 'ArcanistLinter', 'AutoPEP8FormatLinter' => 'ArcanistExternalLinter', 'BashShebangLinter' => 'ArcanistLinter', 'BoostDependenciesLinter' => 'GlobalExternalLinter', 'CHeaderLinter' => 'ArcanistLinter', 'CheckDocLinter' => 'GlobalExternalLinter', 'CheckNonFatalOverAssertInRpc' => 'ArcanistLinter', 'CheckRpcMappingsLinter' => 'GlobalExternalLinter', 'ClangFormatLinter' => 'ArcanistExternalLinter', 'CppCheckLinter' => 'ArcanistExternalLinter', 'CppVoidParameterLinter' => 'ArcanistLinter', 'DoxygenLinter' => 'ArcanistLinter', 'ExtendedConfigurationDrivenLintEngine' => 'ArcanistLintEngine', 'FileNameLinter' => 'ArcanistLinter', 'FormatStringLinter' => 'ArcanistExternalLinter', 'GlobalExternalLinter' => array( 'ArcanistExternalLinter', 'ILintOnce', ), 'IncludeGuardLinter' => 'ArcanistLinter', 'IncludeQuotesLinter' => 'ArcanistLinter', 'IncludeSourceLinter' => 'ArcanistLinter', 'LocaleDependenceLinter' => 'ArcanistLinter', 'LogLinter' => 'ArcanistLinter', + 'MarkdownLinter' => 'ArcanistLinter', 'PythonFileEncodingLinter' => 'ArcanistLinter', 'PythonFormatLinter' => 'ArcanistExternalLinter', 'PythonShebangLinter' => 'ArcanistLinter', 'Qt5Linter' => 'ArcanistLinter', 'ShellCheckLinter' => 'ArcanistExternalLinter', 'ShellLocaleLinter' => 'ArcanistLinter', 'StdintLinter' => 'ArcanistLinter', 'TestsLinter' => 'ArcanistExternalLinter', 'WhitespaceLinter' => 'ArcanistLinter', 'YamllintLinter' => 'ArcanistExternalLinter', ), )); diff --git a/arcanist/linter/MarkdownLinter.php b/arcanist/linter/MarkdownLinter.php new file mode 100644 index 000000000..5d5d7f563 --- /dev/null +++ b/arcanist/linter/MarkdownLinter.php @@ -0,0 +1,95 @@ + ArcanistLintSeverity::SEVERITY_ERROR); + } + + public function getLintNameMap() { + return array( + self::DEAD_LINK_FOUND => pht('Dead link')); + } + + private function isValidLink($currentPath, $link) { + /* + * File anchors are not tested and always considered valid. + * TODO: check that anchors have a corresponding section. + */ + if ($link[0] === '#') { + return true; + } + + /* + * External URLs (starting with scheme://) are considered valid if the URL + * syntax is valid. + */ + if (preg_match('#^[^:]+://#', $link)) { + return filter_var($link, FILTER_VALIDATE_URL); + } + + /* For mailto: links, check the address syntax is valid */ + if (startsWith($link, 'mailto:')) { + return filter_var(substr($link, 7), FILTER_VALIDATE_EMAIL); + } + + if ($link[0] === '/') { + $linkPath = Filesystem::resolvePath( + substr($link, 1), $this->getProjectRoot()); + } else { + $linkPath = Filesystem::resolvePath($link, + pathinfo($currentPath, PATHINFO_DIRNAME)); + } + + return file_exists($linkPath); + } + + public function lintPath($path) { + $path = Filesystem::resolvePath($path, $this->getProjectRoot()); + $fileContent = Filesystem::readFile($path); + + /* Check for broken links (typos in URL of missing target file) */ + $pattern = '/\[[^\]]+\]\(([^)]+\.[^ "#)]+)#?[^)]*\)/'; + if (preg_match_all($pattern, $fileContent, $matches, PREG_OFFSET_CAPTURE)) { + foreach ($matches[1] as $match) { + list($link, $offset) = $match; + + if (!$this->isValidLink($path, trim($link))) { + $this->raiseLintAtOffset( + $offset, + self::DEAD_LINK_FOUND, + pht( + 'The link is invalid, check for syntax error or missing target'), + $link, + null); + } + } + } + } +} diff --git a/contrib/README.md b/contrib/README.md index cf75c5359..2b22cb083 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -1,47 +1,47 @@ Repository Tools --------------------- ### [Developer tools](/contrib/devtools) ### Specific tools for developers working on this repository. Contains the script `github-merge.py` for merging GitHub pull requests securely and signing them using GPG. ### [Verify-Commits](/contrib/verify-commits) ### Tool to verify that every merge commit was signed by a developer using the above `github-merge.py` script. ### [Linearize](/contrib/linearize) ### Construct a linear, no-fork, best version of the blockchain. ### [Qos](/contrib/qos) ### A Linux bash script that will set up traffic control (tc) to limit the outgoing bandwidth for connections to the Bitcoin network. This means one can have an always-on bitcoind instance running, and another local bitcoind/bitcoin-qt instance which connects to this node and receives blocks from it. ### [Seeds](/contrib/seeds) ### Utility to generate the pnSeed[] array that is compiled into the client. Build Tools and Keys --------------------- ### [Debian](/contrib/debian) ### Contains files used to package bitcoind/bitcoin-qt for Debian-based Linux systems. If you compile bitcoind/bitcoin-qt yourself, there are some useful files here. ### [Gitian-descriptors](/contrib/gitian-descriptors) ### Notes on getting Gitian builds up and running using KVM. ### [Gitian-keys](/contrib/gitian-keys) PGP keys used for signing Bitcoin Core [Gitian release](/doc/release-process.md) results. ### [MacDeploy](/contrib/macdeploy) ### Scripts and notes for Mac builds. -### [Gitian-build](/contrib/gitian-build.sh) ### +### [Gitian-build](/contrib/gitian-build.py) ### Script for running full Gitian builds. Test and Verify Tools --------------------- ### [TestGen](/contrib/testgen) ### Utilities to generate test vectors for the data-driven Bitcoin tests. ### [Verify Binaries](/contrib/verifybinaries) ### This script attempts to download and verify the signature file SHA256SUMS.asc from bitcoin.org. diff --git a/contrib/teamcity/README.md b/contrib/teamcity/README.md index 4ef0e502f..ca6e01f4b 100644 --- a/contrib/teamcity/README.md +++ b/contrib/teamcity/README.md @@ -1,84 +1,84 @@ # TeamCity Contrib This directory contains scripts and other files necessary for running builds on the Bitcoin-ABC TeamCity agent instance. # TeamCity Agent Setup Basic steps for creating a new TeamCity build agent. ## Create a new, empty VM Select the following options: 1. OS: Debian (Stretch; Ubuntu works as a backup option) 2. Disk Space: 20GB (Disk space can be increased later. Always start with the smallest disk space necessary to reduce overhead costs) 3. CPUs: 4+ (recommended 8+ for IBD and gitian builds) 4. Memory: 8GB+ (minimum 32GB for running IBD) ## Setup user to run builds ``` # As root, install sudo apt-get install sudo # Still as root, create the build user... adduser teamcity # .. and add it to the sudo group adduser teamcity sudo # Login to this user for the next step sudo su teamcity cd ~ ``` ## Install TeamCity Agent Software In general, the instructions provided by TeamCity may be followed for setting up a new agent image: https://confluence.jetbrains.com/display/TCD18/Setting+up+and+Running+Additional+Build+Agents However, the instructions are not step-by-step and require a lot of decision making. Below is an attempt to break the instructions down so they can be executed quickly: ``` # Install Java JRE sudo apt-get install default-jre sudo vim /etc/environment # Add the following line to /etc/environment: JAVA_HOME="/usr/lib/jvm/default-java" # If this link is not available, it can be found at: # https://build.bitcoinabc.org/agents.html -> "Install Build Agents" wget https://build.bitcoinabc.org/update/buildAgent.zip # Location of where to unzip this is up to you, here buildAgent is assumed: sudo apt-get install unzip unzip -d buildAgent buildAgent.zip cd buildAgent/conf cp buildAgent.dist.properties buildAgent.properties vim buildAgent.properties # Edit the serverUrl line to the following: serverUrl=https://build.bitcoinabc.org/ # Make sure agent.sh is executable cd ../bin chmod +x agent.sh # Setup automatic start for the TeamCity agent cd /etc/init.d sudo vim buildAgent # Copy the contents of the buildAgent-autostart script in the directory of # this README into buildAgent. Modify the AGENT_PATH and/or USER as needed. sudo chmod 755 buildAgent sudo update-rc.d buildAgent defaults # Reboot the machine or manually start the service /etc/init.d/buildAgent start # Ensure the agent is RUNNING /etc/init.d/buildAgent status ``` ## Install necessary build dependencies for bitcoin-abc -See [build-unix.md](doc/build-unix.md) +See [build-unix.md](/doc/build-unix.md) diff --git a/doc/README.md b/doc/README.md index 45ca1e899..ca58b63de 100644 --- a/doc/README.md +++ b/doc/README.md @@ -1,111 +1,110 @@ Bitcoin ABC ===================== Setup --------------------- Bitcoin ABC is a fork of Bitcoin Core, which is the original Bitcoin client and builds the backbone of the network. It downloads and, by default, stores the entire history of Bitcoin transactions, which requires a few hundred gigabytes of disk space. Depending on the speed of your computer and network connection, the synchronization process can take anywhere from a few hours to a day or more. To download Bitcoin ABC, visit [bitcoinabc.org](https://download.bitcoinabc.org/). Verify --------------------- If you download the associated signature files with the binaries from the above link, you can verify the integrity of the binaries by following these instructions, replacing VERSION with the value relevant to you: Get the keys for versions 0.20.11 or later: ``` VERSION="0.20.11" URL="https://download.bitcoinabc.org/${VERSION}/src/bitcoin-abc-${VERSION}.tar.gz" KEYS_FILE="bitcoin-abc-${VERSION}/contrib/gitian-signing/keys.txt" wget -q -O - "${URL}" | tar -zxOf - "${KEYS_FILE}" | while read FINGERPRINT _; do gpg --recv-keys "${FINGERPRINT}"; done ``` Get the keys for versions 0.20.10 or earlier: ``` VERSION="0.20.10" URL="https://raw.githubusercontent.com/Bitcoin-ABC/bitcoin-abc/v${VERSION}/contrib/gitian-signing/keys.txt" wget -q -O - "${URL}" | awk 1 | while read FINGERPRINT _; do gpg --recv-keys "${FINGERPRINT}"; done ``` Check the binaries (all versions): ``` FILE_PATTERN="./*-sha256sums.${VERSION}.asc" gpg --verify-files ${FILE_PATTERN} grep "bitcoin-abc-${VERSION}" ${FILE_PATTERN} | cut -d " " -f 2- | xargs ls 2> /dev/null |\ xargs -i grep -h "{}" ${FILE_PATTERN} | uniq | sha256sum -c ``` *IMPORTANT NOTE:* The first time you run this, all of the signing keys will be UNTRUSTED and you will see warnings indicating this. For best security practices, you should `gpg --sign-key ` for each release signer key and rerun the above script (there should be no warnings the second time). If the keys change unexpectedly, the presence of those warnings should be heeded with extreme caution. Running --------------------- The following are some helpful notes on how to run Bitcoin ABC on your native platform. ### Unix Unpack the files into a directory and run: - `bin/bitcoin-qt` (GUI) or - `bin/bitcoind` (headless) ### Windows Unpack the files into a directory, and then run bitcoin-qt.exe. ### macOS Drag bitcoin-abc to your applications folder, and then run bitcoin-abc. ### Need Help? * See the documentation at the [Bitcoin Wiki](https://en.bitcoin.it/wiki/Main_Page) for help and more information. * Ask for help on the [Bitcoin ABC Subreddit](https://www.reddit.com/r/BitcoinABC/). Building --------------------- The following are developer notes on how to build Bitcoin ABC on your native platform. They are not complete guides, but include notes on the necessary libraries, compile flags, etc. - [Dependencies](dependencies.md) - [macOS Build Notes](build-osx.md) - [Unix Build Notes](build-unix.md) - [Windows Build Notes](build-windows.md) - [Gitian Building Guide](gitian-building.md) Development --------------------- The Bitcoin ABC repo's [root README](/README.md) contains relevant information on the development process and automated testing. - [Developer Notes](developer-notes.md) - [Release Notes](release-notes.md) - [Release Process](release-process.md) - [Source Code Documentation (External Link)](https://dev.visucore.com/bitcoin/doxygen/) - [Translation Process](translation_process.md) - [Translation Strings Policy](translation_strings_policy.md) -- [Travis CI](travis-ci.md) - [Unauthenticated REST Interface](REST-interface.md) - [Shared Libraries](shared-libraries.md) - [BIPS](bips.md) - [Dnsseed Policy](dnsseed-policy.md) - [Benchmarking](benchmarking.md) ### Miscellaneous - [Assets Attribution](assets-attribution.md) - [Files](files.md) - [Fuzz-testing](fuzzing.md) - [Reduce Traffic](reduce-traffic.md) - [Tor Support](tor.md) - [Init Scripts (systemd/upstart/openrc)](init.md) - [ZMQ](zmq.md) License --------------------- Distribution is done under the [MIT software license](/COPYING). This product includes software developed by the OpenSSL Project for use in the [OpenSSL Toolkit](https://www.openssl.org/), cryptographic software written by Eric Young ([eay@cryptsoft.com](mailto:eay@cryptsoft.com)), and UPnP software written by Thomas Bernard. The `gitian-builder` software developed by the Gitian Developers and distributed as part of the project is [licenced](../contrib/gitian-builder/LICENSE) under the [GNU General Public License version 3](../contrib/gitian-builder/COPYING). diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 0de5cd856..3a938b1ae 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -1,1033 +1,1033 @@ Developer Notes =============== **Table of Contents** - [Developer Notes](#developer-notes) - [Coding Style](#coding-style) - [Doxygen comments](#doxygen-comments) - [Development tips and tricks](#development-tips-and-tricks) - [Compiling for debugging](#compiling-for-debugging) - [Compiling for gprof profiling](#compiling-for-gprof-profiling) - [debug.log](#debuglog) - [Writing tests](#writing-tests) - [Writing script integration tests](#writing-script-integration-tests) - [Testnet and Regtest modes](#testnet-and-regtest-modes) - [DEBUG_LOCKORDER](#debug_lockorder) - [Valgrind suppressions file](#valgrind-suppressions-file) - [Compiling for test coverage](#compiling-for-test-coverage) - [Performance profiling with perf](#performance-profiling-with-perf) - [Sanitizers](#sanitizers) - [Locking/mutex usage notes](#lockingmutex-usage-notes) - [Threads](#threads) - [Ignoring IDE/editor files](#ignoring-ideeditor-files) - [Development guidelines](#development-guidelines) - [Wallet](#wallet) - [General C++](#general-c) - [C++ data structures](#c-data-structures) - [Strings and formatting](#strings-and-formatting) - [Variable names](#variable-names) - [Threads and synchronization](#threads-and-synchronization) - [Scripts](#scripts) - [Shebang](#shebang) - [Source code organization](#source-code-organization) - [GUI](#gui) - [Unit tests](#unit-tests) - [Third party libraries](#third-party-libraries) - [Git and GitHub tips](#git-and-github-tips) - [RPC interface guidelines](#rpc-interface-guidelines) Coding Style --------------- Various coding styles have been used during the history of the codebase, and the result is not very consistent. However, we're now trying to converge to a single style, so please use it in new code. Old code will be converted gradually and you are encouraged to use the provided [clang-format-diff script](/contrib/devtools/README.md#clang-format-diffpy) to clean up the patch automatically before submitting a pull request. - Basic rules specified in [src/.clang-format](/src/.clang-format). - Braces on new lines for namespaces, classes, functions, methods. - Braces on the same line for everything else. - 4 space indentation (no tabs) for every block except namespaces. - No indentation for `public`/`protected`/`private` or for `namespace`. - No extra spaces inside parenthesis; don't do ( this ) - No space after function names; one space after `if`, `for` and `while`. - Always add braces for block statements (e.g. `if`, `for`, `while`). - `++i` is preferred over `i++`. - `static_assert` is preferred over `assert` where possible. Generally; compile-time checking is preferred over run-time checking. - Use CamelCase for functions/methods, and lowerCamelCase for variables. - GLOBAL_CONSTANTS should use UPPER_SNAKE_CASE. - namespaces should use lower_snake_case. - Function names should generally start with an English command-form verb (e.g. `ValidateTransaction`, `AddTransactionToMempool`, `ConnectBlock`) - Variable names should generally be nouns or past/future tense verbs. (e.g. `canDoThing`, `signatureOperations`, `didThing`) - Avoid using globals, remove existing globals whenever possible. - Class member variable names should be prepended with `m_` - DO choose easily readable identifier names. - DO favor readability over brevity. - DO NOT use Hungarian notation. - DO NOT use abbreviations or contractions within identifiers. - WRONG: mempool - RIGHT: MemoryPool - WRONG: ChangeDir - RIGHT: ChangeDirectory - DO NOT use obscure acronyms, DO uppercase any acronyms. - FINALLY, do not migrate existing code unless refactoring. It makes forwarding-porting from Bitcoin Core more difficult. The naming convention roughly mirrors [Microsoft Naming Conventions](https://docs.microsoft.com/en-us/dotnet/standard/design-guidelines/general-naming-conventions) C++ Coding Standards should strive to follow the [LLVM Coding Standards](https://llvm.org/docs/CodingStandards.html) Code style example: ```c++ // namespaces should be lower_snake_case namespace foo_bar_bob { /** * Class is used for doing classy things. All classes should * have a doxygen comment describing their PURPOSE. That is to say, * why they exist. Functional details can be determined from the code. * @see PerformTask() */ class Class { private: //! memberVariable's name should be lowerCamelCase, and be a noun. int m_memberVariable; public: /** * The documentation before a function or class method should follow Doxygen * spec. The name of the function should start with an english verb which * indicates the intended purpose of this code. * * The function name should be should be CamelCase. * * @param[in] s A description * @param[in] n Another argument description * @pre Precondition for function... */ bool PerformTask(const std::string& s, int n) { // Use lowerChamelCase for local variables. bool didMore = false; // Comment summarizing the intended purpose of this section of code for (int i = 0; i < n; ++i) { if (!DidSomethingFail()) { return false; } ... if (IsSomethingElse()) { DoMore(); didMore = true; } else { DoLess(); } } return didMore; } } } // namespace foo ``` Doxygen comments ----------------- To facilitate the generation of documentation, use doxygen-compatible comment blocks for functions, methods and fields. For example, to describe a function use: ```c++ /** * ... text ... * @param[in] arg1 A description * @param[in] arg2 Another argument description * @pre Precondition for function... */ bool function(int arg1, const char *arg2) ``` A complete list of `@xxx` commands can be found at http://www.stack.nl/~dimitri/doxygen/manual/commands.html. As Doxygen recognizes the comments by the delimiters (`/**` and `*/` in this case), you don't *need* to provide any commands for a comment to be valid; just a description text is fine. To describe a class use the same construct above the class definition: ```c++ /** * Alerts are for notifying old versions if they become too obsolete and * need to upgrade. The message is displayed in the status bar. * @see GetWarnings() */ class CAlert { ``` To describe a member or variable use: ```c++ int var; //!< Detailed description after the member ``` or ```cpp //! Description before the member int var; ``` Also OK: ```c++ /// /// ... text ... /// bool function2(int arg1, const char *arg2) ``` Not OK (used plenty in the current source, but not picked up): ```c++ // // ... text ... // ``` A full list of comment syntaxes picked up by doxygen can be found at http://www.stack.nl/~dimitri/doxygen/manual/docblocks.html, but if possible use one of the above styles. To build doxygen locally to test changes to the Doxyfile or visualize your comments before landing changes: ``` # In the build directory, call: doxygen doc/Doxyfile # output goes to doc/doxygen/html/ ``` Development tips and tricks --------------------------- ### Compiling for debugging Run `cmake` with `-DCMAKE_BUILD_TYPE=Debug` to add additional compiler flags that produce better debugging builds. ### Compiling for gprof profiling ``` cmake -GNinja .. -DENABLE_HARDENING=OFF -DENABLE_PROFIILING=gprof ``` ### debug.log If the code is behaving strangely, take a look in the debug.log file in the data directory; error and debugging messages are written there. The `-debug=...` command-line option controls debugging; running with just `-debug` or `-debug=1` will turn on all categories (and give you a very large debug.log file). The Qt code routes `qDebug()` output to debug.log under category "qt": run with `-debug=qt` to see it. ### Writing tests For details on unit tests, see `unit-tests.md` For details on functional tests, see `functional-tests.md` ### Writing script integration tests Script integration tests are built using `src/test/script_tests.cpp`: 1. Uncomment the line with `#define UPDATE_JSON_TESTS` 2. Add a new TestBuilder to the `script_build` test to cover your test case. 3. `ninja check-bitcoin-script_tests` 4. Copy your newly generated test JSON from `/src/script_tests.json.gen` to `src/test/data/script_tests.json`. Please commit your TestBuilder along with your generated test JSON and cleanup the uncommented #define before code review. ### Testnet and Regtest modes Run with the `-testnet` option to run with "play bitcoins" on the test network, if you are testing multi-machine code that needs to operate across the internet. If you are testing something that can run on one machine, run with the `-regtest` option. In regression test mode, blocks can be created on-demand; see [test/functional/](/test/functional) for tests that run in `-regtest` mode. ### DEBUG_LOCKORDER Bitcoin ABC is a multi-threaded application, and deadlocks or other multi-threading bugs can be very difficult to track down. The `-DCMAKE_BUILD_TYPE=Debug` cmake option adds `-DDEBUG_LOCKORDER` to the compiler flags. This inserts run-time checks to keep track of which locks are held, and adds warnings to the debug.log file if inconsistencies are detected. ### Valgrind suppressions file Valgrind is a programming tool for memory debugging, memory leak detection, and profiling. The repo contains a Valgrind suppressions file -([`valgrind.supp`](contrib/valgrind.supp)) +([`valgrind.supp`](/contrib/valgrind.supp)) which includes known Valgrind warnings in our dependencies that cannot be fixed in-tree. Example use: ```shell $ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ --show-leak-kinds=all src/test/test_bitcoin --log_level=test_suite $ valgrind -v --leak-check=full src/bitcoind -printtoconsole ``` ### Compiling for test coverage LCOV can be used to generate a test coverage report based upon some test targets execution. Some packages are required to generate the coverage report: `c++filt`, `gcov`, `genhtml`, `lcov` and `python3`. To install these dependencies on Debian 10: ```shell sudo apt install binutils-common g++ lcov python3 ``` To enable LCOV report generation during test runs: ```shell cmake -GNinja .. -DENABLE_COVERAGE=ON ninja coverage-check-all ``` A coverage report will now be accessible at `./check-all.coverage/index.html`. To include branch coverage, you can add the `-DENABLE_BRANCH_COVERAGE=ON` option to the `cmake` command line. ### Performance profiling with perf Profiling is a good way to get a precise idea of where time is being spent in code. One tool for doing profiling on Linux platforms is called [`perf`](http://www.brendangregg.com/perf.html), and has been integrated into the functional test framework. Perf can observe a running process and sample (at some frequency) where its execution is. Perf installation is contingent on which kernel version you're running; see [this StackExchange thread](https://askubuntu.com/questions/50145/how-to-install-perf-monitoring-tool) for specific instructions. Certain kernel parameters may need to be set for perf to be able to inspect the running process' stack. ```sh $ sudo sysctl -w kernel.perf_event_paranoid=-1 $ sudo sysctl -w kernel.kptr_restrict=0 ``` Make sure you [understand the security trade-offs](https://lwn.net/Articles/420403/) of setting these kernel parameters. To profile a running bitcoind process for 60 seconds, you could use an invocation of `perf record` like this: ```sh $ perf record \ -g --call-graph dwarf --per-thread -F 140 \ -p `pgrep bitcoind` -- sleep 60 ``` You could then analyze the results by running ```sh perf report --stdio | c++filt | less ``` or using a graphical tool like [Hotspot](https://github.com/KDAB/hotspot). See the functional test documentation for how to invoke perf within tests. ### Sanitizers Bitcoin ABC can be compiled with various "sanitizers" enabled, which add instrumentation for issues regarding things like memory safety, thread race conditions, or undefined behavior. This is controlled with the `-DENABLE_SANITIZERS` cmake flag, which should be a semicolon separated list of sanitizers to enable. The sanitizer list should correspond to supported `-fsanitize=` options in your compiler. These sanitizers have runtime overhead, so they are most useful when testing changes or producing debugging builds. Some examples: ```bash # Enable both the address sanitizer and the undefined behavior sanitizer cmake -GNinja .. -DENABLE_SANITIZERS="address;undefined" # Enable the thread sanitizer cmake -GNinja .. -DENABLE_SANITIZERS=thread ``` If you are compiling with GCC you will typically need to install corresponding "san" libraries to actually compile with these flags, e.g. libasan for the address sanitizer, libtsan for the thread sanitizer, and libubsan for the undefined sanitizer. If you are missing required libraries, the cmake script will fail with an error when testing the sanitizer flags. Note that the sanitizers will give a better output if they are run with a Debug build configuration. There are a number of known problems for which suppressions files are provided under `test/sanitizer_suppressions`. These files are intended to be used with the `suppressions` option from the sanitizers. If you are using the `check-*` targets to run the tests, the suppression options are automatically set. Otherwise they need to be set manually using environment variables; refer to your compiler manual for the correct syntax. The address sanitizer is known to fail in [sha256_sse4::Transform](/src/crypto/sha256_sse4.cpp) which makes it unusable unless you also use `-DCRYPTO_USE_ASM=OFF` when running cmake. We would like to fix sanitizer issues, so please send pull requests if you can fix any errors found by the address sanitizer (or any other sanitizer). Not all sanitizer options can be enabled at the same time, e.g. trying to build with `-DENABLE_SANITIZERS=="address;thread" will fail in the cmake script as these sanitizers are mutually incompatible. Refer to your compiler manual to learn more about these options and which sanitizers are supported by your compiler. Examples: Build and run the test suite with the address sanitizer enabled: ```bash mkdir build_asan cd build_asan cmake -GNinja .. \ -DCMAKE_BUILD_TYPE=Debug \ -DENABLE_SANITIZERS=address \ -DCRYPTO_USE_ASM=OFF ninja check check-functional ``` Build and run the test suite with the thread sanitizer enabled (it can take a very long time to complete): ```bash mkdir build_tsan cd build_tsan cmake -GNinja .. \ -DCMAKE_BUILD_TYPE=Debug \ -DENABLE_SANITIZERS=thread ninja check check-functional ``` Build and run the test suite with the undefined sanitizer enabled: ```bash mkdir build_ubsan cd build_ubsan cmake -GNinja .. \ -DCMAKE_BUILD_TYPE=Debug \ -DENABLE_SANITIZERS=undefined ninja check check-functional ``` Additional resources: * [AddressSanitizer](https://clang.llvm.org/docs/AddressSanitizer.html) * [LeakSanitizer](https://clang.llvm.org/docs/LeakSanitizer.html) * [MemorySanitizer](https://clang.llvm.org/docs/MemorySanitizer.html) * [ThreadSanitizer](https://clang.llvm.org/docs/ThreadSanitizer.html) * [UndefinedBehaviorSanitizer](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html) * [GCC Instrumentation Options](https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html) * [Google Sanitizers Wiki](https://github.com/google/sanitizers/wiki) * [Issue #12691: Enable -fsanitize flags in Travis](https://github.com/bitcoin/bitcoin/issues/12691) Locking/mutex usage notes ------------------------- The code is multi-threaded, and uses mutexes and the `LOCK` and `TRY_LOCK` macros to protect data structures. Deadlocks due to inconsistent lock ordering (thread 1 locks `cs_main` and then `cs_wallet`, while thread 2 locks them in the opposite order: result, deadlock as each waits for the other to release its lock) are a problem. Compile with `-DDEBUG_LOCKORDER` (or use `-DCMAKE_BUILD_TYPE=Debug`) to get lock order inconsistencies reported in the debug.log file. Re-architecting the core code so there are better-defined interfaces between the various components is a goal, with any necessary locking done by the components (e.g. see the self-contained `FillableSigningProvider` class and its `cs_KeyStore` lock for example). Threads ------- - ThreadScriptCheck : Verifies block scripts. - ThreadImport : Loads blocks from blk*.dat files or bootstrap.dat. - StartNode : Starts other threads. - ThreadDNSAddressSeed : Loads addresses of peers from the DNS. - ThreadMapPort : Universal plug-and-play startup/shutdown - ThreadSocketHandler : Sends/Receives data from peers on port 8333. - ThreadOpenAddedConnections : Opens network connections to added nodes. - ThreadOpenConnections : Initiates new connections to peers. - ThreadMessageHandler : Higher-level message handling (sending and receiving). - DumpAddresses : Dumps IP addresses of nodes to peers.dat. - ThreadRPCServer : Remote procedure call handler, listens on port 8332 for connections and services them. - Shutdown : Does an orderly shutdown of everything. Ignoring IDE/editor files -------------------------- In closed-source environments in which everyone uses the same IDE it is common to add temporary files it produces to the project-wide `.gitignore` file. However, in open source software such as Bitcoin ABC, where everyone uses their own editors/IDE/tools, it is less common. Only you know what files your editor produces and this may change from version to version. The canonical way to do this is thus to create your local gitignore. Add this to `~/.gitconfig`: ``` [core] excludesfile = /home/.../.gitignore_global ``` (alternatively, type the command `git config --global core.excludesfile ~/.gitignore_global` on a terminal) Then put your favorite tool's temporary filenames in that file, e.g. ``` # NetBeans nbproject/ ``` Another option is to create a per-repository excludes file `.git/info/exclude`. These are not committed but apply only to one repository. If a set of tools is used by the build system or scripts the repository (for example, lcov) it is perfectly acceptable to add its files to `.gitignore` and commit them. Development guidelines ============================ A few non-style-related recommendations for developers, as well as points to pay attention to for reviewers of Bitcoin ABC code. Wallet ------- - Make sure that no crashes happen with run-time option `-disablewallet`. - *Rationale*: In RPC code that conditionally uses the wallet (such as `validateaddress`) it is easy to forget that global pointer `pwalletMain` can be NULL. See `test/functional/disablewallet.py` for functional tests exercising the API with `-disablewallet` - Include `db_cxx.h` (BerkeleyDB header) only when `ENABLE_WALLET` is set - *Rationale*: Otherwise compilation of the disable-wallet build will fail in environments without BerkeleyDB General C++ ------------- - Assertions should not have side-effects - *Rationale*: Even though the source code is set to refuse to compile with assertions disabled, having side-effects in assertions is unexpected and makes the code harder to understand - If you use the `.h`, you must link the `.cpp` - *Rationale*: Include files define the interface for the code in implementation files. Including one but not linking the other is confusing. Please avoid that. Moving functions from the `.h` to the `.cpp` should not result in build errors - Use the RAII (Resource Acquisition Is Initialization) paradigm where possible. For example by using `unique_ptr` for allocations in a function. - *Rationale*: This avoids memory and resource leaks, and ensures exception safety - Use `std::make_unique()` to construct objects owned by `unique_ptr`s - *Rationale*: `std::make_unique` is concise and ensures exception safety in complex expressions. C++ data structures -------------------- - Never use the `std::map []` syntax when reading from a map, but instead use `.find()` - *Rationale*: `[]` does an insert (of the default element) if the item doesn't exist in the map yet. This has resulted in memory leaks in the past, as well as race conditions (expecting read-read behavior). Using `[]` is fine for *writing* to a map - Do not compare an iterator from one data structure with an iterator of another data structure (even if of the same type) - *Rationale*: Behavior is undefined. In C++ parlor this means "may reformat the universe", in practice this has resulted in at least one hard-to-debug crash bug - Watch out for out-of-bounds vector access. `&vch[vch.size()]` is illegal, including `&vch[0]` for an empty vector. Use `vch.data()` and `vch.data() + vch.size()` instead. - Vector bounds checking is only enabled in debug mode. Do not rely on it - Make sure that constructors initialize all fields. If this is skipped for a good reason (i.e., optimization on the critical path), add an explicit comment about this - *Rationale*: Ensure determinism by avoiding accidental use of uninitialized values. Also, static analyzers balk about this. - By default, declare single-argument constructors `explicit`. - *Rationale*: This is a precaution to avoid unintended conversions that might arise when single-argument constructors are used as implicit conversion functions. - Use explicitly signed or unsigned `char`s, or even better `uint8_t` and `int8_t`. Do not use bare `char` unless it is to pass to a third-party API. This type can be signed or unsigned depending on the architecture, which can lead to interoperability problems or dangerous conditions such as out-of-bounds array accesses - Prefer explicit constructions over implicit ones that rely on 'magical' C++ behavior - *Rationale*: Easier to understand what is happening, thus easier to spot mistakes, even for those that are not language lawyers - Initialize all non-static class members where they are defined - *Rationale*: Initializing the members in the declaration makes it easy to spot uninitialized ones, and avoids accidentally reading uninitialized memory ```cpp class A { uint32_t m_count{0}; } ``` Strings and formatting ------------------------ - Use `std::string`, avoid C string manipulation functions - *Rationale*: C++ string handling is marginally safer, less scope for buffer overflows and surprises with `\0` characters. Also some C string manipulations tend to act differently depending on platform, or even the user locale - Use `ParseInt32`, `ParseInt64`, `ParseUInt32`, `ParseUInt64`, `ParseDouble` from `utilstrencodings.h` for number parsing - *Rationale*: These functions do overflow checking, and avoid pesky locale issues Variable names -------------- The shadowing warning (`-Wshadow`) is enabled by default. It prevents issues rising from using a different variable with the same name. E.g. in member initializers, prepend `_` to the argument name shadowing the member name: ```c++ class AddressBookPage { Mode m_mode; } AddressBookPage::AddressBookPage(Mode _mode) : m_mode(_mode) ... ``` When using nested cycles, do not name the inner cycle variable the same as in upper cycle etc. Please name variables so that their names do not shadow variables defined in the source code. Threads and synchronization ---------------------------- - Build and run tests with `-DDEBUG_LOCKORDER` to verify that no potential deadlocks are introduced. As of 0.12, this is defined by default when configuring with `-DCMAKE_BUILD_TYPE=Debug` - When using `LOCK`/`TRY_LOCK` be aware that the lock exists in the context of the current scope, so surround the statement and the code that needs the lock with braces OK: ```c++ { TRY_LOCK(cs_vNodes, lockNodes); ... } ``` Wrong: ```c++ TRY_LOCK(cs_vNodes, lockNodes); { ... } ``` Scripts -------------------------- ### Shebang - Use `#!/usr/bin/env bash` instead of obsolete `#!/bin/bash`. - [*Rationale*](https://github.com/dylanaraps/pure-bash-bible#shebang): `#!/bin/bash` assumes it is always installed to /bin/ which can cause issues; `#!/usr/bin/env bash` searches the user's PATH to find the bash binary. OK: ```bash #!/usr/bin/env bash ``` Wrong: ```bash #!/bin/bash ``` Source code organization -------------------------- - Implementation code should go into the `.cpp` file and not the `.h`, unless necessary due to template usage or when performance due to inlining is critical - *Rationale*: Shorter and simpler header files are easier to read, and reduce compile time - Use only the lowercase alphanumerics (`a-z0-9`), underscore (`_`) and hyphen (`-`) in source code filenames. - *Rationale*: `grep`:ing and auto-completing filenames is easier when using a consistent naming pattern. Potential problems when building on case-insensitive filesystems are avoided when using only lowercase characters in source code filenames. - Don't import anything into the global namespace (`using namespace ...`). Use fully specified types such as `std::string`. - *Rationale*: Avoids symbol conflicts - Terminate namespaces with a comment (`// namespace mynamespace`). The comment should be placed on the same line as the brace closing the namespace, e.g. ```c++ namespace mynamespace { ... } // namespace mynamespace namespace { ... } // namespace ``` - *Rationale*: Avoids confusion about the namespace context Header Inclusions ----------------- - Header inclusions should use angle brackets (`#include <>`). The include path should be relative to the `src` folder. e.g.: `#include ` - Native C++ headers should be preferred over C compatibility headers. e.g.: use `` instead of `` - In order to make the code consistent, header files should be included in the following order, with each section separated by a newline: 1. In a .cpp file, the associated .h is in first position. In a test source, this is the header file under test. 2. The project headers. 3. The test headers. 4. The 3rd party libraries headers. Different libraries should be in different sections. 5. The system libraries. All headers should be lexically ordered inside their block. - Use include guards to avoid the problem of double inclusion. The header file `foo/bar.h` should use the include guard identifier `BITCOIN_FOO_BAR_H`, e.g. ```c++ #ifndef BITCOIN_FOO_BAR_H #define BITCOIN_FOO_BAR_H ... #endif // BITCOIN_FOO_BAR_H ``` GUI ----- - Do not display or manipulate dialogs in model code (classes `*Model`) - *Rationale*: Model classes pass through events and data from the core, they should not interact with the user. That's where View classes come in. The converse also holds: try to not directly access core data structures from Views. - Avoid adding slow or blocking code in the GUI thread. In particular do not add new `interface::Node` and `interface::Wallet` method calls, even if they may be fast now, in case they are changed to lock or communicate across processes in the future. Prefer to offload work from the GUI thread to worker threads (see `RPCExecutor` in console code as an example) or take other steps (see https://doc.qt.io/archives/qq/qq27-responsive-guis.html) to keep the GUI responsive. - *Rationale*: Blocking the GUI thread can increase latency, and lead to hangs and deadlocks. Unit Tests ----------- - Test suite naming convention: The Boost test suite in file `src/test/foo_tests.cpp` should be named `foo_tests`. Test suite names must be unique. Third party libraries --------------------- Several parts of the repository are software maintained elsewhere. Changes to these should preferably be sent upstream but bugfixes may also be submitted to Bitcoin ABC so that they can be integrated quickly. Cosmetic changes should be purely taken upstream. Current third party libraries include: - src/leveldb - Upstream at https://github.com/google/leveldb ; Maintained by Google. - **Note**: Follow the instructions in [Upgrading LevelDB](#upgrading-leveldb) when merging upstream changes to Bitcoin ABC. - src/libsecp256k1 - Upstream at https://github.com/bitcoin-core/secp256k1/ ; actively maintained by Bitcoin Core contributors. Bitcoin ABC is using a modified version of libsecp256k1, some changes might be directly submitted to Bitcoin ABC. See the [secp256k1 README](/src/secp256k1/README.md) for details. - src/crypto/ctaes - Upstream at https://github.com/bitcoin-core/ctaes ; maintained by Bitcoin Core contributors. - src/univalue - Upstream at https://github.com/jgarzik/univalue ; maintained by Jeff Garzik. Upgrading LevelDB --------------------- Extra care must be taken when upgrading LevelDB. This section explains issues you must be aware of. ### File Descriptor Counts In most configurations we use the default LevelDB value for `max_open_files`, which is 1000 at the time of this writing. If LevelDB actually uses this many file descriptors it will cause problems with Bitcoin's `select()` loop, because it may cause new sockets to be created where the fd value is >= 1024. For this reason, on 64-bit Unix systems we rely on an internal LevelDB optimization that uses `mmap()` + `close()` to open table files without actually retaining references to the table file descriptors. If you are upgrading LevelDB, you must sanity check the changes to make sure that this assumption remains valid. In addition to reviewing the upstream changes in `env_posix.cc`, you can use `lsof` to check this. For example, on Linux this command will show open `.ldb` file counts: ```bash $ lsof -p $(pidof bitcoind) |\ awk 'BEGIN { fd=0; mem=0; } /ldb$/ { if ($4 == "mem") mem++; else fd++ } END { printf "mem = %s, fd = %s\n", mem, fd}' mem = 119, fd = 0 ``` The `mem` value shows how many files are mmap'ed, and the `fd` value shows you many file descriptors these files are using. You should check that `fd` is a small number (usually 0 on 64-bit hosts). See the notes in the `SetMaxOpenFiles()` function in `dbwrapper.cc` for more details. ### Consensus Compatibility It is possible for LevelDB changes to inadvertently change consensus compatibility between nodes. This happened in Bitcoin 0.8 (when LevelDB was first introduced). When upgrading LevelDB you should review the upstream changes to check for issues affecting consensus compatibility. For example, if LevelDB had a bug that accidentally prevented a key from being returned in an edge case, and that bug was fixed upstream, the bug "fix" would be an incompatible consensus change. In this situation the correct behavior would be to revert the upstream fix before applying the updates to Bitcoin ABC's copy of LevelDB. In general you should be wary of any upstream changes affecting what data is returned from LevelDB queries. Git and GitHub tips --------------------- - Github is not typically the source of truth for pull requests. See CONTRIBUTING.md for instructions on setting up your repo correctly. - Similarly, your git remote origin should be set to: `ssh://vcs@reviews.bitcoinabc.org:2221/source/bitcoin-abc.git` instead of github.com. See CONTRIBUTING.md for details. - For resolving merge/rebase conflicts, it can be useful to enable diff3 style using `git config merge.conflictstyle diff3`. Instead of <<< yours === theirs >>> you will see <<< yours ||| original === theirs >>> This may make it much clearer what caused the conflict. In this style, you can often just look at what changed between *original* and *theirs*, and mechanically apply that to *yours* (or the other way around). - When reviewing patches which change indentation in C++ files, use `git diff -w` and `git show -w`. This makes the diff algorithm ignore whitespace changes. This feature is also available on github.com, by adding `?w=1` at the end of any URL which shows a diff. - When reviewing patches that change symbol names in many places, use `git diff --word-diff`. This will instead of showing the patch as deleted/added *lines*, show deleted/added *words*. - When reviewing patches that move code around, try using `git diff --patience commit~:old/file.cpp commit:new/file/name.cpp`, and ignoring everything except the moved body of code which should show up as neither `+` or `-` lines. In case it was not a pure move, this may even work when combined with the `-w` or `--word-diff` options described above. - When looking at other's pull requests, it may make sense to add the following section to your `.git/config` file: [remote "upstream-pull"] fetch = +refs/pull/*:refs/remotes/upstream-pull/* url = git@github.com:bitcoin/bitcoin.git This will add an `upstream-pull` remote to your git repository, which can be fetched using `git fetch --all` or `git fetch upstream-pull`. Afterwards, you can use `upstream-pull/NUMBER/head` in arguments to `git show`, `git checkout` and anywhere a commit id would be acceptable to see the changes from pull request NUMBER. RPC interface guidelines -------------------------- A few guidelines for introducing and reviewing new RPC interfaces: - Method naming: use consecutive lower-case names such as `getrawtransaction` and `submitblock` - *Rationale*: Consistency with existing interface. - Argument naming: use snake case `fee_delta` (and not, e.g. camel case `feeDelta`) - *Rationale*: Consistency with existing interface. - Use the JSON parser for parsing, don't manually parse integers or strings from arguments unless absolutely necessary. - *Rationale*: Introduces hand-rolled string manipulation code at both the caller and callee sites, which is error prone, and it is easy to get things such as escaping wrong. JSON already supports nested data structures, no need to re-invent the wheel. - *Exception*: AmountFromValue can parse amounts as string. This was introduced because many JSON parsers and formatters hard-code handling decimal numbers as floating point values, resulting in potential loss of precision. This is unacceptable for monetary values. **Always** use `AmountFromValue` and `ValueFromAmount` when inputting or outputting monetary values. The only exceptions to this are `prioritisetransaction` and `getblocktemplate` because their interface is specified as-is in BIP22. - Missing arguments and 'null' should be treated the same: as default values. If there is no default value, both cases should fail in the same way. The easiest way to follow this guideline is detect unspecified arguments with `params[x].isNull()` instead of `params.size() <= x`. The former returns true if the argument is either null or missing, while the latter returns true if is missing, and false if it is null. - *Rationale*: Avoids surprises when switching to name-based arguments. Missing name-based arguments are passed as 'null'. - Try not to overload methods on argument type. E.g. don't make `getblock(true)` and `getblock("hash")` do different things. - *Rationale*: This is impossible to use with `bitcoin-cli`, and can be surprising to users. - *Exception*: Some RPC calls can take both an `int` and `bool`, most notably when a bool was switched to a multi-value, or due to other historical reasons. **Always** have false map to 0 and true to 1 in this case. - Don't forget to fill in the argument names correctly in the RPC command table. - *Rationale*: If not, the call can not be used with name-based arguments. - Set okSafeMode in the RPC command table to a sensible value: safe mode is when the blockchain is regarded to be in a confused state, and the client deems it unsafe to do anything irreversible such as send. Anything that just queries should be permitted. - *Rationale*: Troubleshooting a node in safe mode is difficult if half the RPCs don't work. - Add every non-string RPC argument `(method, idx, name)` to the table `vRPCConvertParams` in `rpc/client.cpp`. - *Rationale*: `bitcoin-cli` and the GUI debug console use this table to determine how to convert a plaintext command line to JSON. If the types don't match, the method can be unusable from there. - A RPC method must either be a wallet method or a non-wallet method. Do not introduce new methods such as `signrawtransaction` that differ in behavior based on presence of a wallet. - *Rationale*: as well as complicating the implementation and interfering with the introduction of multi-wallet, wallet and non-wallet code should be separated to avoid introducing circular dependencies between code units. - Try to make the RPC response a JSON object. - *Rationale*: If a RPC response is not a JSON object then it is harder to avoid API breakage if new data in the response is needed. - Wallet RPCs call BlockUntilSyncedToCurrentChain to maintain consistency with `getblockchaininfo`'s state immediately prior to the call's execution. Wallet RPCs whose behavior does *not* depend on the current chainstate may omit this call. - *Rationale*: In previous versions of Bitcoin Core, the wallet was always in-sync with the chainstate (by virtue of them all being updated in the same cs_main lock). In order to maintain the behavior that wallet RPCs return results as of at least the highest best-known block an RPC client may be aware of prior to entering a wallet RPC call, we must block until the wallet is caught up to the chainstate as of the RPC call's entry. This also makes the API much easier for RPC clients to reason about. - Be aware of RPC method aliases and generally avoid registering the same callback function pointer for different RPCs. - *Rationale*: RPC methods registered with the same function pointer will be considered aliases and only the first method name will show up in the `help` rpc command list. - *Exception*: Using RPC method aliases may be appropriate in cases where a new RPC is replacing a deprecated RPC, to avoid both RPCs confusingly showing up in the command list. diff --git a/doc/functional-tests.md b/doc/functional-tests.md index 02488d5a6..79fb70d53 100644 --- a/doc/functional-tests.md +++ b/doc/functional-tests.md @@ -1,347 +1,347 @@ # Functional tests The [/test/](/test/) directory contains integration tests that test bitcoind and its utilities in their entirety. It does not contain unit tests, which can be found in [/src/test](/src/test), [/src/wallet/test](/src/wallet/test), etc. There are currently two sets of tests in the [/test/](/test/) directory: - [functional](/test/functional) which test the functionality of bitcoind and bitcoin-qt by interacting with them through the RPC and P2P interfaces. - [util](/test/util) which tests the bitcoin utilities, currently only bitcoin-tx. The util tests are run as part of `make check` target. The functional tests are run by the Teamcity continuous build process whenever a diff is created or updated on Phabricator. Both sets of tests can also be run locally. # Running functional tests locally Build for your system first. Be sure to enable wallet, utils and daemon when you configure. Tests will not run otherwise. ### Functional tests #### Dependencies The ZMQ functional test requires a python ZMQ library. To install it: - On Unix, run `sudo apt-get install python3-zmq` - On mac OS, run `pip3 install pyzmq` #### Running the tests Individual tests can be run by directly calling the test script, eg: ``` test/functional/example_test.py ``` or can be run through the test_runner harness, eg: ``` test/functional/test_runner.py example_test ``` You can run any combination (incl. duplicates) of tests by calling: ``` test/functional/test_runner.py ... ``` Run the regression test suite with: ``` test/functional/test_runner.py ``` Run all possible tests with ``` test/functional/test_runner.py --extended ``` By default, up to 4 tests will be run in parallel by test_runner. To specify how many jobs to run, append `--jobs=n` The individual tests and the test_runner harness have many command-line options. Run `test_runner.py -h` to see them all. #### Troubleshooting and debugging test failures ##### Resource contention The P2P and RPC ports used by the bitcoind nodes-under-test are chosen to make conflicts with other processes unlikely. However, if there is another bitcoind process running on the system (perhaps from a previous test which hasn't successfully killed all its bitcoind nodes), then there may be a port conflict which will cause the test to fail. It is recommended that you run the tests on a system where no other bitcoind processes are running. On linux, the test_framework will warn if there is another bitcoind process running when the tests are started. If there are zombie bitcoind processes after test failure, you can kill them by running the following commands. **Note that these commands will kill all bitcoind processes running on the system, so should not be used if any non-test bitcoind processes are being run.** ```bash killall bitcoind ``` or ```bash pkill -9 bitcoind ``` ##### Data directory cache A pre-mined blockchain with 200 blocks is generated the first time a functional test is run and is stored in test/cache. This speeds up test startup times since new blockchains don't need to be generated for each test. However, the cache may get into a bad state, in which case tests will fail. If this happens, remove the cache directory (and make sure bitcoind processes are stopped as above): ```bash rm -rf cache killall bitcoind ``` ##### Test logging The tests contain logging at different levels (debug, info, warning, etc). By default: - When run through the test_runner harness, *all* logs are written to `test_framework.log` and no logs are output to the console. - When run directly, *all* logs are written to `test_framework.log` and INFO level and above are output to the console. - When run on Travis, no logs are output to the console. However, if a test fails, the `test_framework.log` and bitcoind `debug.log`s will all be dumped to the console to help troubleshooting. To change the level of logs output to the console, use the `-l` command line argument. `test_framework.log` and bitcoind `debug.log`s can be combined into a single aggregate log by running the `combine_logs.py` script. The output can be plain text, colorized text or html. For example: ``` combine_logs.py -c | less -r ``` will pipe the colorized logs from the test into less. Use `--tracerpc` to trace out all the RPC calls and responses to the console. For some tests (eg any that use `submitblock` to submit a full block over RPC), this can result in a lot of screen output. By default, the test data directory will be deleted after a successful run. Use `--nocleanup` to leave the test data directory intact. The test data directory is never deleted after a failed test. ##### Attaching a debugger A python debugger can be attached to tests at any point. Just add the line: ```py import pdb; pdb.set_trace() ``` anywhere in the test. You will then be able to inspect variables, as well as call methods that interact with the bitcoind nodes-under-test. If further introspection of the bitcoind instances themselves becomes necessary, this can be accomplished by first setting a pdb breakpoint at an appropriate location, running the test to that point, then using `gdb` to attach to the process and debug. For instance, to attach to `self.node[1]` during a run: ```bash 2017-06-27 14:13:56.686000 TestFramework (INFO): Initializing test directory /tmp/user/1000/testo9vsdjo3 ``` use the directory path to get the pid from the pid file: ```bash cat /tmp/user/1000/testo9vsdjo3/node1/regtest/bitcoind.pid gdb /home/example/bitcoind ``` Note: gdb attach step may require `sudo`. To get rid of this, you can run: ```bash echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope ``` ### Benchmarking and profiling with perf An easy way to profile node performance during functional tests is provided for Linux platforms using `perf`. Perf will sample the running node and will generate profile data in the node's datadir. The profile data can then be presented using `perf report` or a graphical tool like [hotspot](https://github.com/KDAB/hotspot). There are two ways of invoking perf: one is to use the `--perf` flag when running tests, which will profile each node during the entire test run: perf begins to profile when the node starts and ends when it shuts down. The other way is the use the `profile_with_perf` context manager, e.g. ```python with node.profile_with_perf("send-big-msgs"): # Perform activity on the node you're interested in profiling, e.g.: for _ in range(10000): node.p2p.send_message(some_large_message) ``` To see useful textual output, run ```sh perf report -i /path/to/datadir/send-big-msgs.perf.data.xxxx --stdio | c++filt | less ``` #### See also: - [Installing perf](https://askubuntu.com/q/50145) - [Perf examples](http://www.brendangregg.com/perf.html) - [Hotspot](https://github.com/KDAB/hotspot): a GUI for perf output analysis ##### Prevent using deprecated features Python will issue a `DeprecationWarning` when a deprecated feature is encountered in a script. By default, this warning message is ignored and not displayed to the user. This behavior can be changed by setting the environment variable `PYTHONWARNINGS` as follow: `PYTHONWARNINGS=default::DeprecationWarning` The warning message will now be printed to the `sys.stderr` output. ### Util tests Util tests can be run locally by running `test/util/bitcoin-util-test.py`. Use the `-v` option for verbose output. # Writing functional tests #### Example test -The [example_test.py](example_test.py) is a heavily commented example of a test -case that uses both the RPC and P2P interfaces. If you are writing your first -test, copy that file and modify to fit your needs. +The [example_test.py](/test/functional/example_test.py) is a heavily commented +example of a test case that uses both the RPC and P2P interfaces. If you are +writing your first test, copy that file and modify to fit your needs. #### Coverage Running `test_runner.py` with the `--coverage` argument tracks which RPCs are called by the tests and prints a report of uncovered RPCs in the summary. This can be used (along with the `--extended` argument) to find out which RPCs we don't have test cases for. #### Style guidelines - Where possible, try to adhere to [PEP-8 guidelines](https://www.python.org/dev/peps/pep-0008/) - Use a python linter like flake8 before submitting PRs to catch common style nits (eg trailing whitespace, unused imports, etc) - Avoid wildcard imports where possible - Use a module-level docstring to describe what the test is testing, and how it is testing it. - When subclassing the BitcoinTestFramwork, place overrides for the `set_test_params()`, `add_options()` and `setup_xxxx()` methods at the top of the subclass, then locally-defined helper methods, then the `run_test()` method. #### Naming guidelines - Name the test `_test.py`, where area can be one of the following: - `feature` for tests for full features that aren't wallet/mining/mempool, eg `feature_rbf.py` - `interface` for tests for other interfaces (REST, ZMQ, etc), eg `interface_rest.py` - `mempool` for tests for mempool behaviour, eg `mempool_reorg.py` - `mining` for tests for mining features, eg `mining_prioritisetransaction.py` - `p2p` for tests that explicitly test the p2p interface, eg `p2p_disconnect_ban.py` - `rpc` for tests for individual RPC methods or features, eg `rpc_listtransactions.py` - `wallet` for tests for wallet features, eg `wallet_keypool.py` - use an underscore to separate words - exception: for tests for specific RPCs or command line options which don't include underscores, name the test after the exact RPC or argument name, eg `rpc_decodescript.py`, not `rpc_decode_script.py` - Don't use the redundant word `test` in the name, eg `interface_zmq.py`, not `interface_zmq_test.py` #### General test-writing advice - Set `self.num_nodes` to the minimum number of nodes necessary for the test. Having additional unrequired nodes adds to the execution time of the test as well as memory/CPU/disk requirements (which is important when running tests in parallel or on Travis). - Avoid stop-starting the nodes multiple times during the test if possible. A stop-start takes several seconds, so doing it several times blows up the runtime of the test. - Set the `self.setup_clean_chain` variable in `set_test_params()` to control whether or not to use the cached data directories. The cached data directories contain a 200-block pre-mined blockchain and wallets for four nodes. Each node has 25 mature blocks (25x50=1250 BTC) in its wallet. - When calling RPCs with lots of arguments, consider using named keyword arguments instead of positional arguments to make the intent of the call clear to readers. - Many of the core test framework classes such as `CBlock` and `CTransaction` don't allow new attributes to be added to their objects at runtime like typical Python objects allow. This helps prevent unpredictable side effects from typographical errors or usage of the objects outside of their intended purpose. #### RPC and P2P definitions Test writers may find it helpful to refer to the definitions for the RPC and P2P messages. These can be found in the following source files: - `/src/rpc/*` for RPCs - `/src/wallet/rpc*` for wallet RPCs - `ProcessMessage()` in `/src/net_processing.cpp` for parsing P2P messages #### Using the P2P interface - `messages.py` contains all the definitions for objects that pass over the network (`CBlock`, `CTransaction`, etc, along with the network-level wrappers for them, `msg_block`, `msg_tx`, etc). - P2P tests have two threads. One thread handles all network communication with the bitcoind(s) being tested in a callback-based event loop; the other implements the test logic. - `P2PConnection` is the class used to connect to a bitcoind. `P2PInterface` contains the higher level logic for processing P2P payloads and connecting to the Bitcoin Core node application logic. For custom behaviour, subclass the P2PInterface object and override the callback methods. - Can be used to write tests where specific P2P protocol behavior is tested. Examples tests are `p2p-acceptblock.py`, `p2p-compactblocks.py`. ### test-framework modules -#### [test_framework/authproxy.py](test_framework/authproxy.py) +#### [test_framework/authproxy.py](/test/functional/test_framework/authproxy.py) Taken from the [python-bitcoinrpc repository](https://github.com/jgarzik/python-bitcoinrpc). -#### [test_framework/test_framework.py](test_framework/test_framework.py) +#### [test_framework/test_framework.py](/test/functional/test_framework/test_framework.py) Base class for functional tests. -#### [test_framework/util.py](test_framework/util.py) +#### [test_framework/util.py](/test/functional/test_framework/util.py) Generally useful functions. -#### [test_framework/mininode.py](test_framework/mininode.py) +#### [test_framework/mininode.py](/test/functional/test_framework/mininode.py) Basic code to support P2P connectivity to a bitcoind. -#### [test_framework/script.py](test_framework/script.py) +#### [test_framework/script.py](/test/functional/test_framework/script.py) Utilities for manipulating transaction scripts (originally from python-bitcoinlib) -#### [test_framework/key.py](test_framework/key.py) +#### [test_framework/key.py](/test/functional/test_framework/key.py) Wrapper around OpenSSL EC_Key (originally from python-bitcoinlib) -#### [test_framework/bignum.py](test_framework/bignum.py) +#### [test_framework/bignum.py](/test/functional/test_framework/bignum.py) Helpers for script.py -#### [test_framework/blocktools.py](test_framework/blocktools.py) +#### [test_framework/blocktools.py](/test/functional/test_framework/blocktools.py) Helper functions for creating blocks and transactions. diff --git a/doc/release-process.md b/doc/release-process.md index 29506ab35..4fccd3ff4 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -1,61 +1,61 @@ Bitcoin ABC Release Process =========================== ## Before Release 1. Check feature completeness - Check features planned for the release are implemented and documented. Any incomplete items should be discussed with the Release Manager as soon as possible. - Known bugs should have tickets - Any known issues or limitations should be documented in release notes 2. Update the documents / code which needs to be updated every release - - Check that [release-notes.md](doc/release-notes.md) is complete, and fill in any missing items. + - Check that [release-notes.md](/doc/release-notes.md) is complete, and fill in any missing items. - Verify the following were updated by automation since the last release: - - Seeds (see [README](contrib/seeds/README.md)) + - Seeds (see [README](/contrib/seeds/README.md)) - Chainparams were updated, such as assume-valid, chainwork, and disk size expectations. 3. Verify tests passed - Run `arc lint --everything` and check there is no linter error - Ensure that bitcoind and bitcoin-qt run with no issue on all supported platforms. Manually test bitcoin-qt by sending some transactions and navigating through the menus. 4. Add git tag for release a. Create the tag: `git tag vM.m.r` (M = major version, m = minor version, r = revision) b. Push the tag to Github: `git push vM.m.r` 5. Increment version number for the next release in: - `doc/release-notes.md` (and copy existing one to versioned `doc/release-notes/*.md`) - `CMakeLists.txt` - `contrib/seeds/makeseeds.py` (only after a new major release) ## Release 6. Create Gitian Builds (see [gitian-building.md](/doc/gitian-building.md)) 7. Verify matching Gitian Builds, gather signatures 8. Verify IBD both with and without `-checkpoints=0 -assumevalid=0` 9. Upload Gitian Builds to [bitcoinabc.org](https://download.bitcoinabc.org/) 10. Create a [GitHub release](https://github.com/Bitcoin-ABC/bitcoin-abc/releases): `contrib/release/github-release.sh -a -t -o ` 11. Create [Ubuntu PPA packages](https://launchpad.net/~bitcoin-abc/+archive/ubuntu/ppa): `contrib/release/debian-packages.sh ` 12. Notify maintainers of AUR and Docker images to build their packages. They should be given 1-day advance notice if possible. ## After Release 13. Publish signed checksums (various places, e.g. blog, reddit/r/BitcoinABC) 14. Announce Release: - [Reddit](https://www.reddit.com/r/BitcoinABC/) - Twitter @Bitcoin_ABC - Public slack channels friendly to Bitcoin ABC announcements (eg. #abc-announce on BTCforks, #hardfork on BTCchat)