diff --git a/.arclint b/.arclint --- a/.arclint +++ b/.arclint @@ -142,6 +142,13 @@ "exclude": [ "(__init__\\.py$)" ] + }, + "shellcheck": { + "type": "shellcheck", + "include": "(\\.sh$)", + "exclude": [ + "(^src/(secp256k1|univalue)/)" + ] } } } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,7 +127,7 @@ During submission of patches, arcanist will automatically run `arc lint` to enforce Bitcoin ABC code formatting standards, and often suggests changes. If code formatting tools do not install automatically on your system, you -will have to install clang-format-7, autopep8, flake8 and phpcs. +will have to install clang-format-7, autopep8, flake8, phpcs and shellcheck. To install clang-format-7 on Ubuntu (>= 18.04+updates) or Debian (>= 10): ``` @@ -138,7 +138,7 @@ To install autopep8, flake8 and phpcs on Ubuntu: ``` -sudo apt-get install python-autopep8 flake8 php-codesniffer +sudo apt-get install python-autopep8 flake8 php-codesniffer shellcheck ``` diff --git a/arcanist/.phutil_module_cache b/arcanist/.phutil_module_cache --- a/arcanist/.phutil_module_cache +++ b/arcanist/.phutil_module_cache @@ -1 +1 @@ -{"__symbol_cache_version__":11,"ae5b7d3d6b8cf9598ce4abaf0cd56b21":{"have":{"class":{"ExtendedConfigurationDrivenLintEngine":19}},"need":{"function":{"newv":159,"pht":933},"class":{"ArcanistLintEngine":65,"ArcanistConfigurationDrivenLintEngine":171,"PhutilConsole":866},"class\/interface":{"ILintOnce":634}},"xmap":{"ExtendedConfigurationDrivenLintEngine":["ArcanistLintEngine"]}},"2809b09d2021203b43c57da33d1fe8bf":{"have":{"class":{"AssertWithSideEffectsLinter":210}},"need":{"function":{"pht":439},"class":{"ArcanistLinter":246,"ArcanistLintSeverity":926,"Filesystem":1170}},"xmap":{"AssertWithSideEffectsLinter":["ArcanistLinter"]}},"90a8b110dc475955f15bb81d37268cb5":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"execx":769,"id":1903},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1910,"Filesystem":1754,"ArcanistLinter":2017,"ArcanistLintSeverity":2095}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"5ea58c19df0397ed8ee0f463d90d6c72":{"have":{"class":{"BoostDependenciesLinter":145}},"need":{"function":{"pht":330,"id":1440},"class":{"GlobalExternalLinter":177,"ArcanistLintMessage":1447,"Filesystem":609,"ArcanistLinter":1524,"ArcanistLintSeverity":1624}},"xmap":{"BoostDependenciesLinter":["GlobalExternalLinter"]}},"38f0c676bff5192a344464142caaa253":{"have":{"class":{"CHeaderLinter":99}},"need":{"function":{"pht":611},"class":{"ArcanistLinter":121,"ArcanistLintSeverity":1060,"Filesystem":1307}},"xmap":{"CHeaderLinter":["ArcanistLinter"]}},"9bb48ec0fe2e9ced8e27d42540d0571c":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":321,"id":1845},"class":{"GlobalExternalLinter":129,"ArcanistLintMessage":1852,"Filesystem":729,"ArcanistLinter":1900,"ArcanistLintSeverity":1986}},"xmap":{"CheckDocLinter":["GlobalExternalLinter"]}},"63d19a8745cb2e1200cc26488dc7ad25":{"have":{"class":{"CheckRpcMappingsLinter":131}},"need":{"function":{"pht":310,"id":1386},"class":{"GlobalExternalLinter":162,"ArcanistLintMessage":1393,"Filesystem":573,"ArcanistLinter":1443,"ArcanistLintSeverity":1544}},"xmap":{"CheckRpcMappingsLinter":["GlobalExternalLinter"]}},"75579a609dd975aa0226add52700c622":{"have":{"class":{"FileNameLinter":103}},"need":{"function":{"pht":307},"class":{"ArcanistLinter":126,"ArcanistLintSeverity":662,"Filesystem":968}},"xmap":{"FileNameLinter":["ArcanistLinter"]}},"9285ad9415f8ebe564f7119e5a72c559":{"have":{"class":{"FormatStringLinter":146}},"need":{"function":{"pht":377,"csprintf":1492,"id":1872},"class":{"ArcanistExternalLinter":173,"ArcanistLintMessage":1879,"Filesystem":827,"ArcanistLinter":1956,"ArcanistLintSeverity":2044}},"xmap":{"FormatStringLinter":["ArcanistExternalLinter"]}},"2e11dd9ad67e594f863bc46ac59ea37e":{"have":{"class":{"GlobalExternalLinter":199}},"need":{"class":{"ArcanistExternalLinter":228},"interface":{"ILintOnce":262}},"xmap":{"GlobalExternalLinter":["ArcanistExternalLinter","ILintOnce"]}},"b2403124ec3e8be6cb4d10bf0f6c4134":{"have":{"interface":{"ILintOnce":69}},"need":[],"xmap":[]},"1f9bac7956f4f948a187828dcc6ba2d0":{"have":{"class":{"IncludeGuardLinter":98}},"need":{"function":{"pht":368},"class":{"ArcanistLinter":125,"ArcanistLintSeverity":721,"Filesystem":970}},"xmap":{"IncludeGuardLinter":["ArcanistLinter"]}},"2cbb6e7228d81557f777ad648704f343":{"have":{"class":{"IncludeQuotesLinter":100}},"need":{"function":{"pht":306},"class":{"ArcanistLinter":128,"ArcanistLintSeverity":663,"Filesystem":964}},"xmap":{"IncludeQuotesLinter":["ArcanistLinter"]}},"f151089cf79fdb8257b2272ed4782d88":{"have":{"class":{"IncludeSourceLinter":99}},"need":{"function":{"pht":391},"class":{"ArcanistLinter":127,"ArcanistLintSeverity":699,"Filesystem":938}},"xmap":{"IncludeSourceLinter":["ArcanistLinter"]}},"ce5dee893bedb5d93466655eede8ac47":{"have":{"class":{"LocaleDependenceLinter":160}},"need":{"function":{"pht":5429},"class":{"ArcanistLinter":191,"ArcanistLintSeverity":5932,"Filesystem":6178}},"xmap":{"LocaleDependenceLinter":["ArcanistLinter"]}},"6f2f22dd0f259fb2eaa284b4fab3bc29":{"have":{"class":{"PythonFormatLinter":123}},"need":{"function":{"pht":353,"id":1838},"class":{"ArcanistExternalLinter":150,"ArcanistLintMessage":1845,"Filesystem":776,"ArcanistLinter":1970,"ArcanistLintSeverity":2053}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"03cf226fe177a9fce7dc046816cc466b":{"have":{"class":{"PythonOpenFileEncodingLinter":111}},"need":{"function":{"pht":325},"class":{"ArcanistLinter":148,"ArcanistLintSeverity":687,"Filesystem":991}},"xmap":{"PythonOpenFileEncodingLinter":["ArcanistLinter"]}},"09a933fbbf135320585be52750d93831":{"have":{"class":{"StdintLinter":90}},"need":{"function":{"pht":280},"class":{"ArcanistLinter":111,"ArcanistLintSeverity":589,"Filesystem":897}},"xmap":{"StdintLinter":["ArcanistLinter"]}},"25781df78f6eebfb223296b8265e9d19":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2629},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2636,"Filesystem":776,"ArcanistLinter":2684,"ArcanistLintSeverity":2792}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}},"454160ed72e88797129d84ee589de304":{"have":{"class":{"PythonShebangLinter":87}},"need":{"function":{"pht":290},"class":{"ArcanistLinter":115,"ArcanistLintSeverity":614,"Filesystem":854}},"xmap":{"PythonShebangLinter":["ArcanistLinter"]}},"1c03a4631b02b8e1ebcc3a078dc1e4af":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":781,"id":2235},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":2242,"Filesystem":2086,"ArcanistLinter":2349,"ArcanistLintSeverity":2427}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}}} \ No newline at end of file +{"__symbol_cache_version__":11,"ae5b7d3d6b8cf9598ce4abaf0cd56b21":{"have":{"class":{"ExtendedConfigurationDrivenLintEngine":19}},"need":{"function":{"newv":159,"pht":933},"class":{"ArcanistLintEngine":65,"ArcanistConfigurationDrivenLintEngine":171,"PhutilConsole":866},"class\/interface":{"ILintOnce":634}},"xmap":{"ExtendedConfigurationDrivenLintEngine":["ArcanistLintEngine"]}},"2809b09d2021203b43c57da33d1fe8bf":{"have":{"class":{"AssertWithSideEffectsLinter":210}},"need":{"function":{"pht":439},"class":{"ArcanistLinter":246,"ArcanistLintSeverity":926,"Filesystem":1170}},"xmap":{"AssertWithSideEffectsLinter":["ArcanistLinter"]}},"90a8b110dc475955f15bb81d37268cb5":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"execx":769,"id":1903},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1910,"Filesystem":1754,"ArcanistLinter":2017,"ArcanistLintSeverity":2095}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"5ea58c19df0397ed8ee0f463d90d6c72":{"have":{"class":{"BoostDependenciesLinter":145}},"need":{"function":{"pht":330,"id":1440},"class":{"GlobalExternalLinter":177,"ArcanistLintMessage":1447,"Filesystem":609,"ArcanistLinter":1524,"ArcanistLintSeverity":1624}},"xmap":{"BoostDependenciesLinter":["GlobalExternalLinter"]}},"38f0c676bff5192a344464142caaa253":{"have":{"class":{"CHeaderLinter":99}},"need":{"function":{"pht":611},"class":{"ArcanistLinter":121,"ArcanistLintSeverity":1060,"Filesystem":1307}},"xmap":{"CHeaderLinter":["ArcanistLinter"]}},"9bb48ec0fe2e9ced8e27d42540d0571c":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":321,"id":1845},"class":{"GlobalExternalLinter":129,"ArcanistLintMessage":1852,"Filesystem":729,"ArcanistLinter":1900,"ArcanistLintSeverity":1986}},"xmap":{"CheckDocLinter":["GlobalExternalLinter"]}},"63d19a8745cb2e1200cc26488dc7ad25":{"have":{"class":{"CheckRpcMappingsLinter":131}},"need":{"function":{"pht":310,"id":1386},"class":{"GlobalExternalLinter":162,"ArcanistLintMessage":1393,"Filesystem":573,"ArcanistLinter":1443,"ArcanistLintSeverity":1544}},"xmap":{"CheckRpcMappingsLinter":["GlobalExternalLinter"]}},"1c03a4631b02b8e1ebcc3a078dc1e4af":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":781,"id":2235},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":2242,"Filesystem":2086,"ArcanistLinter":2349,"ArcanistLintSeverity":2427}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}},"75579a609dd975aa0226add52700c622":{"have":{"class":{"FileNameLinter":103}},"need":{"function":{"pht":307},"class":{"ArcanistLinter":126,"ArcanistLintSeverity":662,"Filesystem":968}},"xmap":{"FileNameLinter":["ArcanistLinter"]}},"9285ad9415f8ebe564f7119e5a72c559":{"have":{"class":{"FormatStringLinter":146}},"need":{"function":{"pht":377,"csprintf":1492,"id":1872},"class":{"ArcanistExternalLinter":173,"ArcanistLintMessage":1879,"Filesystem":827,"ArcanistLinter":1956,"ArcanistLintSeverity":2044}},"xmap":{"FormatStringLinter":["ArcanistExternalLinter"]}},"2e11dd9ad67e594f863bc46ac59ea37e":{"have":{"class":{"GlobalExternalLinter":199}},"need":{"class":{"ArcanistExternalLinter":228},"interface":{"ILintOnce":262}},"xmap":{"GlobalExternalLinter":["ArcanistExternalLinter","ILintOnce"]}},"b2403124ec3e8be6cb4d10bf0f6c4134":{"have":{"interface":{"ILintOnce":69}},"need":[],"xmap":[]},"1f9bac7956f4f948a187828dcc6ba2d0":{"have":{"class":{"IncludeGuardLinter":98}},"need":{"function":{"pht":368},"class":{"ArcanistLinter":125,"ArcanistLintSeverity":721,"Filesystem":970}},"xmap":{"IncludeGuardLinter":["ArcanistLinter"]}},"2cbb6e7228d81557f777ad648704f343":{"have":{"class":{"IncludeQuotesLinter":100}},"need":{"function":{"pht":306},"class":{"ArcanistLinter":128,"ArcanistLintSeverity":663,"Filesystem":964}},"xmap":{"IncludeQuotesLinter":["ArcanistLinter"]}},"f151089cf79fdb8257b2272ed4782d88":{"have":{"class":{"IncludeSourceLinter":99}},"need":{"function":{"pht":391},"class":{"ArcanistLinter":127,"ArcanistLintSeverity":699,"Filesystem":938}},"xmap":{"IncludeSourceLinter":["ArcanistLinter"]}},"6f2f22dd0f259fb2eaa284b4fab3bc29":{"have":{"class":{"PythonFormatLinter":123}},"need":{"function":{"pht":353,"id":1838},"class":{"ArcanistExternalLinter":150,"ArcanistLintMessage":1845,"Filesystem":776,"ArcanistLinter":1970,"ArcanistLintSeverity":2053}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"03cf226fe177a9fce7dc046816cc466b":{"have":{"class":{"PythonOpenFileEncodingLinter":111}},"need":{"function":{"pht":325},"class":{"ArcanistLinter":148,"ArcanistLintSeverity":687,"Filesystem":991}},"xmap":{"PythonOpenFileEncodingLinter":["ArcanistLinter"]}},"454160ed72e88797129d84ee589de304":{"have":{"class":{"PythonShebangLinter":87}},"need":{"function":{"pht":290},"class":{"ArcanistLinter":115,"ArcanistLintSeverity":614,"Filesystem":854}},"xmap":{"PythonShebangLinter":["ArcanistLinter"]}},"09a933fbbf135320585be52750d93831":{"have":{"class":{"StdintLinter":90}},"need":{"function":{"pht":280},"class":{"ArcanistLinter":111,"ArcanistLintSeverity":589,"Filesystem":897}},"xmap":{"StdintLinter":["ArcanistLinter"]}},"25781df78f6eebfb223296b8265e9d19":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2629},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2636,"Filesystem":776,"ArcanistLinter":2684,"ArcanistLintSeverity":2792}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}},"899b8be78da0be470965682eaa31f00a":{"have":{"class":{"ShellCheckLinter":95}},"need":{"function":{"pht":1835,"id":3051},"class":{"ArcanistExternalLinter":120,"ArcanistLintMessage":3058,"ArcanistLintSeverity":1560,"ArcanistLinter":3106}},"xmap":{"ShellCheckLinter":["ArcanistExternalLinter"]}},"8a0f2ba683e96feeb9d9564af01fe3e7":{"have":{"class":{"LocaleDependenceLinter":160}},"need":{"function":{"pht":5460},"class":{"ArcanistLinter":191,"ArcanistLintSeverity":5963,"Filesystem":6209}},"xmap":{"LocaleDependenceLinter":["ArcanistLinter"]}}} \ No newline at end of file diff --git a/arcanist/__phutil_library_map__.php b/arcanist/__phutil_library_map__.php --- a/arcanist/__phutil_library_map__.php +++ b/arcanist/__phutil_library_map__.php @@ -28,6 +28,7 @@ 'PythonFormatLinter' => 'linter/PythonFormatLinter.php', 'PythonOpenFileEncodingLinter' => 'linter/PythonOpenFileEncodingLinter.php', 'PythonShebangLinter' => 'linter/PythonShebangLinter.php', + 'ShellCheckLinter' => 'linter/ShellCheckLinter.php', 'StdintLinter' => 'linter/StdintLinter.php', 'TestsLinter' => 'linter/TestsLinter.php', ), @@ -54,6 +55,7 @@ 'PythonFormatLinter' => 'ArcanistExternalLinter', 'PythonOpenFileEncodingLinter' => 'ArcanistLinter', 'PythonShebangLinter' => 'ArcanistLinter', + 'ShellCheckLinter' => 'ArcanistExternalLinter', 'StdintLinter' => 'ArcanistLinter', 'TestsLinter' => 'ArcanistExternalLinter', ), diff --git a/arcanist/linter/ShellCheckLinter.php b/arcanist/linter/ShellCheckLinter.php new file mode 100644 --- /dev/null +++ b/arcanist/linter/ShellCheckLinter.php @@ -0,0 +1,99 @@ + ArcanistLintSeverity::SEVERITY_ADVICE, + 'warning' => ArcanistLintSeverity::SEVERITY_WARNING, + 'error' => ArcanistLintSeverity::SEVERITY_ERROR, + ); + + public function getInfoName() { + return 'shellcheck'; + } + + public function getInfoDescription() { + return pht('Use shellcheck for processing specified files.'); + } + + public function getLinterName() { + return 'SHELLCHECK'; + } + + public function getLinterConfigurationName() { + return 'shellcheck'; + } + + public function getDefaultBinary() { + return 'shellcheck'; + } + + public function getInstallInstructions() { + return pht('Make sure shellcheck is in directory specified by $PATH'); + } + + public function shouldExpectCommandErrors() { + return true; + } + + protected function getMandatoryFlags() { + return array( + '--format=gcc', + '--exclude='.implode(',', self::SHELLCHECK_EXCLUDED_RULES)); + } + + private function getSeverity($severity) { + if (array_key_exists($severity, self::SHELLCHECK_SEVERITY_MAP)) { + return self::SHELLCHECK_SEVERITY_MAP[$severity]; + } + + return ArcanistLintSeverity::SEVERITY_ERROR; + } + + protected function parseLinterOutput($path, $err, $stdout, $stderr) { + $messages = array(); + + $pattern = '/(.+):(\d+):(\d+): (.+): (.+) \[(SC\d+)\]/'; + if (preg_match_all($pattern, $stdout, $errors, PREG_SET_ORDER)) { + foreach ($errors as $error) { + list(, $file, $line, $char, $severity, $message, $code) = $error; + + $messages[] = id(new ArcanistLintMessage()) + ->setGranularity(ArcanistLinter::GRANULARITY_FILE) + ->setPath($path) + ->setLine($line) + ->setChar($char) + ->setCode($code) + ->setSeverity($this->getSeverity($severity)) + ->setName('ShellCheck found an issue:') + ->setDescription($message); + } + } + + return $messages; + } +} diff --git a/contrib/gitian-signing/check-keys.sh b/contrib/gitian-signing/check-keys.sh --- a/contrib/gitian-signing/check-keys.sh +++ b/contrib/gitian-signing/check-keys.sh @@ -1,5 +1,5 @@ # Fetch latest signers' keys. We do this in order to check if a key was revoked. -while read fingerprint keyholder +while read fingerprint _ do gpg --recv-keys ${fingerprint} done < ./keys.txt diff --git a/contrib/gitian/provisioner.sh b/contrib/gitian/provisioner.sh --- a/contrib/gitian/provisioner.sh +++ b/contrib/gitian/provisioner.sh @@ -14,17 +14,21 @@ # make sure that USE_LXC is always set when logging in as vagrant, # and configure LXC IP addresses -echo 'export USE_LXC=1' >> /home/${BUILDUSER}/.profile -echo 'export LXC_BRIDGE=lxcbr0' >> /home/${BUILDUSER}/.profile -echo 'export GITIAN_HOST_IP=10.0.3.1' >> /home/${BUILDUSER}/.profile -echo 'export LXC_GUEST_IP=10.0.3.5' >> /home/${BUILDUSER}/.profile +{ + echo 'export USE_LXC=1' + echo 'export LXC_BRIDGE=lxcbr0' + echo 'export GITIAN_HOST_IP=10.0.3.1' + echo 'export LXC_GUEST_IP=10.0.3.5' +} >> /home/${BUILDUSER}/.profile # Setup bridge echo 'USE_LXC_BRIDGE="true"' > /etc/default/lxc-net -echo 'lxc.network.type = veth' > /etc/lxc/default.conf -echo 'lxc.network.link = lxcbr0' >> /etc/lxc/default.conf -echo 'lxc.network.flags = up' >> /etc/lxc/default.conf -echo 'lxc.network.hwaddr = 00:16:3e:xx:xx:xx' >> /etc/lxc/default.conf +{ + echo 'lxc.network.type = veth' + echo 'lxc.network.link = lxcbr0' + echo 'lxc.network.flags = up' + echo 'lxc.network.hwaddr = 00:16:3e:xx:xx:xx' +} > /etc/lxc/default.conf service lxc-net restart # chdir into build user home directory diff --git a/contrib/teamcity/build.sh b/contrib/teamcity/build.sh --- a/contrib/teamcity/build.sh +++ b/contrib/teamcity/build.sh @@ -39,7 +39,7 @@ # Create the log directory if it doesn't exist and clear it mkdir -p "${SAN_LOG_DIR}" -rm -rf "${SAN_LOG_DIR}"/* +rm -rf "${SAN_LOG_DIR:?}"/* # Sanitizers options, not used if sanitizers are not enabled export ASAN_OPTIONS="malloc_context_size=0:log_path=${SAN_LOG_DIR}/asan.log" diff --git a/contrib/teamcity/gitian.sh b/contrib/teamcity/gitian.sh --- a/contrib/teamcity/gitian.sh +++ b/contrib/teamcity/gitian.sh @@ -2,8 +2,10 @@ cd "$(dirname "$0")" -export COMMIT=`git rev-parse HEAD` -export PROJECT_ROOT=`git rev-parse --show-toplevel` +COMMIT=`git rev-parse HEAD` +export COMMIT +PROJECT_ROOT=`git rev-parse --show-toplevel` +export PROJECT_ROOT export USE_LXC=1 export GITIAN_HOST_IP=10.0.3.1 export LXC_BRIDGE=lxcbr0