diff --git a/.arclint b/.arclint index ba251eae6..124b06f28 100644 --- a/.arclint +++ b/.arclint @@ -1,60 +1,58 @@ { "linters": { "generated": { "type": "generated" }, "clang-format": { "type": "clang-format", "version": "7.0", "bin": ["clang-format-7", "clang-format"], "include": "(^src/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "autopep8": { "type": "autopep8", "version": ">=1.3.4", "include": "(\\.py$)" }, "flake8": { "type": "flake8", "include": "(\\.py$)", "flags": [ "--select=F401,F403,F405" ] }, "lint-format-strings": { - "type": "script-and-regex", + "type": "lint-format-strings", "include": "(^src/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" - ], - "script-and-regex.script": "test/lint/lint-format-strings.sh", - "script-and-regex.regex": "/^(?P.+): (?P.+:.+)$/m" + ] }, "check-doc": { "type": "check-doc", "include": "(^src/.*\\.(h|c|cpp)$)" }, "lint-tests": { "type": "lint-tests", "include": "(^src/(rpc/|wallet/)?test/.*\\.(cpp)$)" }, "lint-python-format": { "type": "lint-python-format", "include": "(\\.py$)", "exclude": [ "(^test/lint/lint-python-format\\.py$)" ] }, "phpcs": { "type": "phpcs", "include": "(\\.php$)", "exclude": [ "(^arcanist/__phutil_library_.+\\.php$)" ], "phpcs.standard": "arcanist/linter/phpcs_ruleset.xml" } } } diff --git a/arcanist/.phutil_module_cache b/arcanist/.phutil_module_cache index ff60f64e1..3182b7fc5 100644 --- a/arcanist/.phutil_module_cache +++ b/arcanist/.phutil_module_cache @@ -1 +1 @@ -{"__symbol_cache_version__":11,"d60c8224f471e0ecddc2a6f3c6839cd1":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"id":1317},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1324,"Filesystem":1168,"ArcanistLinter":1431,"ArcanistLintSeverity":1509}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"213c3145da34ed6dfc0d70d628a2a086":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":323,"id":1868},"class":{"ArcanistExternalLinter":129,"ArcanistLintMessage":1875,"Filesystem":737,"ArcanistLinter":1923,"ArcanistLintSeverity":2009}},"xmap":{"CheckDocLinter":["ArcanistExternalLinter"]}},"7bab1f879b8a86dd9977b8c0d075935f":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":787,"id":1664},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":1671,"Filesystem":1515,"ArcanistLinter":1778,"ArcanistLintSeverity":1856}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}},"0e068a1116ed03e86a2388020a821983":{"have":{"class":{"PythonFormatLinter":75}},"need":{"function":{"pht":305,"id":1614},"class":{"ArcanistExternalLinter":102,"ArcanistLintMessage":1621,"Filesystem":733,"ArcanistLinter":1752,"ArcanistLintSeverity":1835}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"ea2beb1668dfbdd87488f18fbb20178f":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2676},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2683,"Filesystem":791,"ArcanistLinter":2731,"ArcanistLintSeverity":2839}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}}} \ No newline at end of file +{"__symbol_cache_version__":11,"90a8b110dc475955f15bb81d37268cb5":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"execx":769,"id":1903},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1910,"Filesystem":1754,"ArcanistLinter":2017,"ArcanistLintSeverity":2095}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"bf0805c02029a7226e8c0d7dee039b3c":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":323,"id":1847},"class":{"ArcanistExternalLinter":129,"ArcanistLintMessage":1854,"Filesystem":731,"ArcanistLinter":1902,"ArcanistLintSeverity":1988}},"xmap":{"CheckDocLinter":["ArcanistExternalLinter"]}},"6af7410cfea496ff1d4dcc2624b6b8ea":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":781,"id":1653},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":1660,"Filesystem":1504,"ArcanistLinter":1767,"ArcanistLintSeverity":1845}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}},"6f2f22dd0f259fb2eaa284b4fab3bc29":{"have":{"class":{"PythonFormatLinter":123}},"need":{"function":{"pht":353,"id":1838},"class":{"ArcanistExternalLinter":150,"ArcanistLintMessage":1845,"Filesystem":776,"ArcanistLinter":1970,"ArcanistLintSeverity":2053}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"25781df78f6eebfb223296b8265e9d19":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2629},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2636,"Filesystem":776,"ArcanistLinter":2684,"ArcanistLintSeverity":2792}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}},"9285ad9415f8ebe564f7119e5a72c559":{"have":{"class":{"FormatStringLinter":146}},"need":{"function":{"pht":377,"csprintf":1492,"id":1872},"class":{"ArcanistExternalLinter":173,"ArcanistLintMessage":1879,"Filesystem":827,"ArcanistLinter":1956,"ArcanistLintSeverity":2044}},"xmap":{"FormatStringLinter":["ArcanistExternalLinter"]}}} \ No newline at end of file diff --git a/arcanist/__phutil_library_map__.php b/arcanist/__phutil_library_map__.php index 82f69981f..cab17f1d4 100644 --- a/arcanist/__phutil_library_map__.php +++ b/arcanist/__phutil_library_map__.php @@ -1,26 +1,28 @@ 2, 'class' => array( 'AutoPEP8FormatLinter' => 'linter/AutoPEP8Linter.php', 'CheckDocLinter' => 'linter/CheckDocLinter.php', 'ClangFormatLinter' => 'linter/ClangFormatLinter.php', + 'FormatStringLinter' => 'linter/FormatStringLinter.php', 'PythonFormatLinter' => 'linter/PythonFormatLinter.php', 'TestsLinter' => 'linter/TestsLinter.php', ), 'function' => array(), 'xmap' => array( 'AutoPEP8FormatLinter' => 'ArcanistExternalLinter', 'CheckDocLinter' => 'ArcanistExternalLinter', 'ClangFormatLinter' => 'ArcanistExternalLinter', + 'FormatStringLinter' => 'ArcanistExternalLinter', 'PythonFormatLinter' => 'ArcanistExternalLinter', 'TestsLinter' => 'ArcanistExternalLinter', ), )); diff --git a/arcanist/linter/FormatStringLinter.php b/arcanist/linter/FormatStringLinter.php new file mode 100644 index 000000000..52d687c9b --- /dev/null +++ b/arcanist/linter/FormatStringLinter.php @@ -0,0 +1,86 @@ +getProjectRoot()); + } + + public function shouldUseInterpreter() { + return true; + } + + public function getDefaultInterpreter() { + return "python3"; + } + + public function getInstallInstructions() { + return pht('The test/lint/lint-format-strings.py script is part of the '. + 'bitcoin-abc project'); + } + + public function shouldExpectCommandErrors() { + return false; + } + + protected function getMandatoryFlags() { + return array(); + } + + protected function getPathArgumentForLinterFuture($path) { + // The path is expected to be relative to the project root + return csprintf('%s', + Filesystem::readablePath($path, $this->getProjectRoot())); + } + + protected function parseLinterOutput($path, $err, $stdout, $stderr) { + $pattern = '/^(?P.+): (?P.+): (?P.+)$/m'; + + $messages = array(); + + if (preg_match_all($pattern, $stdout, $matches, PREG_SET_ORDER)) { + foreach($matches as $match) { + $messages[] = id(new ArcanistLintMessage()) + ->setPath($path) + ->setGranularity(ArcanistLinter::GRANULARITY_FILE) + ->setCode('STRFMT') + ->setSeverity(ArcanistLintSeverity::SEVERITY_ERROR) + ->setName('String formatting function arguments mismatch') + ->setDescription($match["code"]."\n".$match["message"]); + } + } + + return $messages; + } +} diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index 550c84374..fc090b23d 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -1,293 +1,289 @@ #!/usr/bin/env python3 # # Copyright (c) 2018 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # # Lint format strings: This program checks that the number of arguments passed # to a variadic format string function matches the number of format specifiers # in the format string. import argparse +import doctest import re import sys FALSE_POSITIVES = [ ("src/dbwrapper.cpp", "vsnprintf(p, limit - p, format, backup_ap)"), ("src/index/base.cpp", "FatalError(const char* fmt, const Args&... args)"), ("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"), ("src/util.cpp", "strprintf(_(COPYRIGHT_HOLDERS), _(COPYRIGHT_HOLDERS_SUBSTITUTION))"), ("src/util.cpp", "strprintf(COPYRIGHT_HOLDERS, COPYRIGHT_HOLDERS_SUBSTITUTION)"), ("src/seeder/main.cpp", "fprintf(stderr, help, argv[0])"), ("src/tinyformat.h", "printf(const char *fmt, const Args &... args)"), ("src/tinyformat.h", "printf(const char *fmt, TINYFORMAT_VARARGS(n))"), ] FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS = [ ("FatalError", 0), ("fprintf", 1), ("LogConnectFailure", 1), ("LogPrint", 1), ("LogPrintf", 0), ("printf", 0), ("snprintf", 2), ("sprintf", 1), ("strprintf", 0), ("vfprintf", 1), ("vprintf", 1), ("vsnprintf", 1), ("vsprintf", 1), ] def parse_function_calls(function_name, source_code): """Return an array with all calls to function function_name in string source_code. Preprocessor directives and C++ style comments ("//") in source_code are removed. >>> len(parse_function_calls("foo", "foo();bar();foo();bar();")) 2 >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[0].startswith("foo(1);") True >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[1].startswith("foo(2);") True >>> len(parse_function_calls("foo", "foo();bar();// foo();bar();")) 1 >>> len(parse_function_calls("foo", "#define FOO foo();")) 0 """ assert(type(function_name) is str and type( source_code) is str and function_name) lines = [re.sub("// .*", " ", line).strip() for line in source_code.split("\n") if not line.strip().startswith("#")] return re.findall(r"[^a-zA-Z_](?=({}\(.*).*)".format(function_name), " " + " ".join(lines)) def normalize(s): """Return a normalized version of string s with newlines, tabs and C style comments ("/* ... */") replaced with spaces. Multiple spaces are replaced with a single space. >>> normalize(" /* nothing */ foo\tfoo /* bar */ foo ") 'foo foo foo' """ assert(type(s) is str) s = s.replace("\n", " ") s = s.replace("\t", " ") s = re.sub("/\*.*?\*/", " ", s) s = re.sub(" {2,}", " ", s) return s.strip() ESCAPE_MAP = { r"\n": "[escaped-newline]", r"\t": "[escaped-tab]", r'\"': "[escaped-quote]", } def escape(s): """Return the escaped version of string s with "\\\"", "\\n" and "\\t" escaped as "[escaped-backslash]", "[escaped-newline]" and "[escaped-tab]". >>> unescape(escape("foo")) == "foo" True >>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"') 'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]' """ assert(type(s) is str) for raw_value, escaped_value in ESCAPE_MAP.items(): s = s.replace(raw_value, escaped_value) return s def unescape(s): """Return the unescaped version of escaped string s. Reverses the replacements made in function escape(s). >>> unescape(escape("bar")) 'bar' >>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]") 'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"' """ assert(type(s) is str) for raw_value, escaped_value in ESCAPE_MAP.items(): s = s.replace(escaped_value, raw_value) return s def parse_function_call_and_arguments(function_name, function_call): """Split string function_call into an array of strings consisting of: * the string function_call followed by "(" * the function call argument #1 * ... * the function call argument #n * a trailing ");" The strings returned are in escaped form. See escape(...). >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");') ['foo(', '"%s",', ' "foo"', ')'] >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");') ['foo(', '"%s",', ' "foo"', ')'] >>> parse_function_call_and_arguments("foo", 'foo("%s %s", "foo", "bar");') ['foo(', '"%s %s",', ' "foo",', ' "bar"', ')'] >>> parse_function_call_and_arguments("fooprintf", 'fooprintf("%050d", i);') ['fooprintf(', '"%050d",', ' i', ')'] >>> parse_function_call_and_arguments("foo", 'foo(bar(foobar(barfoo("foo"))), foobar); barfoo') ['foo(', 'bar(foobar(barfoo("foo"))),', ' foobar', ')'] >>> parse_function_call_and_arguments("foo", "foo()") ['foo(', '', ')'] >>> parse_function_call_and_arguments("foo", "foo(123)") ['foo(', '123', ')'] >>> parse_function_call_and_arguments("foo", 'foo("foo")') ['foo(', '"foo"', ')'] """ assert(type(function_name) is str and type( function_call) is str and function_name) remaining = normalize(escape(function_call)) expected_function_call = "{}(".format(function_name) assert(remaining.startswith(expected_function_call)) parts = [expected_function_call] remaining = remaining[len(expected_function_call):] open_parentheses = 1 in_string = False parts.append("") for char in remaining: parts.append(parts.pop() + char) if char == "\"": in_string = not in_string continue if in_string: continue if char == "(": open_parentheses += 1 continue if char == ")": open_parentheses -= 1 if open_parentheses > 1: continue if open_parentheses == 0: parts.append(parts.pop()[:-1]) parts.append(char) break if char == ",": parts.append("") return parts def parse_string_content(argument): """Return the text within quotes in string argument. >>> parse_string_content('1 "foo %d bar" 2') 'foo %d bar' >>> parse_string_content('1 foobar 2') '' >>> parse_string_content('1 "bar" 2') 'bar' >>> parse_string_content('1 "foo" 2 "bar" 3') 'foobar' >>> parse_string_content('1 "foo" 2 " " "bar" 3') 'foo bar' >>> parse_string_content('""') '' >>> parse_string_content('') '' >>> parse_string_content('1 2 3') '' """ assert(type(argument) is str) string_content = "" in_string = False for char in normalize(escape(argument)): if char == "\"": in_string = not in_string elif in_string: string_content += char return string_content def count_format_specifiers(format_string): """Return the number of format specifiers in string format_string. >>> count_format_specifiers("foo bar foo") 0 >>> count_format_specifiers("foo %d bar foo") 1 >>> count_format_specifiers("foo %d bar %i foo") 2 >>> count_format_specifiers("foo %d bar %i foo %% foo") 2 >>> count_format_specifiers("foo %d bar %i foo %% foo %d foo") 3 >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo") 4 """ assert(type(format_string) is str) n = 0 in_specifier = False for i, char in enumerate(format_string): if format_string[i - 1:i + 1] == "%%" or format_string[i:i + 2] == "%%": pass elif char == "%": in_specifier = True n += 1 elif char in "aAcdeEfFgGinopsuxX": in_specifier = False elif in_specifier and char == "*": n += 1 return n def main(args_in): """ Return a string output with information on string format errors >>> main(["test/lint/lint-format-strings-tests.txt"]) test/lint/lint-format-strings-tests.txt: Expected 1 argument(s) after format string but found 2 argument(s): printf("%d", 1, 2) test/lint/lint-format-strings-tests.txt: Expected 2 argument(s) after format string but found 3 argument(s): printf("%a %b", 1, 2, "anything") test/lint/lint-format-strings-tests.txt: Expected 1 argument(s) after format string but found 0 argument(s): printf("%d") test/lint/lint-format-strings-tests.txt: Expected 3 argument(s) after format string but found 2 argument(s): printf("%a%b%z", 1, "anything") - 1 >>> main(["test/lint/lint-format-strings-tests-skip-arguments.txt"]) test/lint/lint-format-strings-tests-skip-arguments.txt: Expected 1 argument(s) after format string but found 2 argument(s): fprintf(skipped, "%d", 1, 2) test/lint/lint-format-strings-tests-skip-arguments.txt: Expected 1 argument(s) after format string but found 0 argument(s): fprintf(skipped, "%d") test/lint/lint-format-strings-tests-skip-arguments.txt: Expected 1 argument(s) after format string but found 2 argument(s): snprintf(skip1, skip2, "%d", 1, 2) test/lint/lint-format-strings-tests-skip-arguments.txt: Expected 1 argument(s) after format string but found 0 argument(s): snprintf(skip1, skip2, "%d") test/lint/lint-format-strings-tests-skip-arguments.txt: Could not parse function call string "snprintf(...)": snprintf(skip1, "%d") - 1 """ parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed " "to a variadic format string function matches the number of format " "specifiers in the format string.") parser.add_argument("file", type=argparse.FileType( "r", encoding="utf-8"), nargs="*", help="C++ source code file (e.g. foo.cpp)") args = parser.parse_args(args_in) - exit_code = 0 for f in args.file: file_content = f.read() for (function_name, skip_arguments) in FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS: for function_call_str in parse_function_calls(function_name, file_content): parts = parse_function_call_and_arguments( function_name, function_call_str) relevant_function_call_str = unescape("".join(parts))[:512] if (f.name, relevant_function_call_str) in FALSE_POSITIVES: continue if len(parts) < 3 + skip_arguments: - exit_code = 1 print("{}: Could not parse function call string \"{}(...)\": {}".format( f.name, function_name, relevant_function_call_str)) continue argument_count = len(parts) - 3 - skip_arguments format_str = parse_string_content(parts[1 + skip_arguments]) format_specifier_count = count_format_specifiers(format_str) if format_specifier_count != argument_count: - exit_code = 1 print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format( f.name, format_specifier_count, argument_count, relevant_function_call_str)) continue - return exit_code if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) + doctest.testmod() + main(sys.argv[1:]) diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh deleted file mode 100755 index 1f56f4818..000000000 --- a/test/lint/lint-format-strings.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Lint format strings: This program checks that the number of arguments passed -# to a variadic format string function matches the number of format specifiers -# in the format string. - -export LC_ALL=C - -EXIT_CODE=0 -if ! python3 -m doctest test/lint/lint-format-strings.py; then - EXIT_CODE=1 -fi - -test/lint/lint-format-strings.py "${1}" - -exit ${EXIT_CODE}