diff --git a/.arclint b/.arclint
index ba251eae6..124b06f28 100644
--- a/.arclint
+++ b/.arclint
@@ -1,60 +1,58 @@
{
"linters": {
"generated": {
"type": "generated"
},
"clang-format": {
"type": "clang-format",
"version": "7.0",
"bin": ["clang-format-7", "clang-format"],
"include": "(^src/.*\\.(h|c|cpp)$)",
"exclude": [
"(^src/(secp256k1|univalue|leveldb)/)"
]
},
"autopep8": {
"type": "autopep8",
"version": ">=1.3.4",
"include": "(\\.py$)"
},
"flake8": {
"type": "flake8",
"include": "(\\.py$)",
"flags": [
"--select=F401,F403,F405"
]
},
"lint-format-strings": {
- "type": "script-and-regex",
+ "type": "lint-format-strings",
"include": "(^src/.*\\.(h|c|cpp)$)",
"exclude": [
"(^src/(secp256k1|univalue|leveldb)/)"
- ],
- "script-and-regex.script": "test/lint/lint-format-strings.sh",
- "script-and-regex.regex": "/^(?P<file>.+): (?P<message>.+:.+)$/m"
+ ]
},
"check-doc": {
"type": "check-doc",
"include": "(^src/.*\\.(h|c|cpp)$)"
},
"lint-tests": {
"type": "lint-tests",
"include": "(^src/(rpc/|wallet/)?test/.*\\.(cpp)$)"
},
"lint-python-format": {
"type": "lint-python-format",
"include": "(\\.py$)",
"exclude": [
"(^test/lint/lint-python-format\\.py$)"
]
},
"phpcs": {
"type": "phpcs",
"include": "(\\.php$)",
"exclude": [
"(^arcanist/__phutil_library_.+\\.php$)"
],
"phpcs.standard": "arcanist/linter/phpcs_ruleset.xml"
}
}
}
diff --git a/arcanist/.phutil_module_cache b/arcanist/.phutil_module_cache
index ff60f64e1..3182b7fc5 100644
--- a/arcanist/.phutil_module_cache
+++ b/arcanist/.phutil_module_cache
@@ -1 +1 @@
-{"__symbol_cache_version__":11,"d60c8224f471e0ecddc2a6f3c6839cd1":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"id":1317},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1324,"Filesystem":1168,"ArcanistLinter":1431,"ArcanistLintSeverity":1509}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"213c3145da34ed6dfc0d70d628a2a086":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":323,"id":1868},"class":{"ArcanistExternalLinter":129,"ArcanistLintMessage":1875,"Filesystem":737,"ArcanistLinter":1923,"ArcanistLintSeverity":2009}},"xmap":{"CheckDocLinter":["ArcanistExternalLinter"]}},"7bab1f879b8a86dd9977b8c0d075935f":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":787,"id":1664},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":1671,"Filesystem":1515,"ArcanistLinter":1778,"ArcanistLintSeverity":1856}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}},"0e068a1116ed03e86a2388020a821983":{"have":{"class":{"PythonFormatLinter":75}},"need":{"function":{"pht":305,"id":1614},"class":{"ArcanistExternalLinter":102,"ArcanistLintMessage":1621,"Filesystem":733,"ArcanistLinter":1752,"ArcanistLintSeverity":1835}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"ea2beb1668dfbdd87488f18fbb20178f":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2676},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2683,"Filesystem":791,"ArcanistLinter":2731,"ArcanistLintSeverity":2839}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}}}
\ No newline at end of file
+{"__symbol_cache_version__":11,"90a8b110dc475955f15bb81d37268cb5":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"execx":769,"id":1903},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1910,"Filesystem":1754,"ArcanistLinter":2017,"ArcanistLintSeverity":2095}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"bf0805c02029a7226e8c0d7dee039b3c":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":323,"id":1847},"class":{"ArcanistExternalLinter":129,"ArcanistLintMessage":1854,"Filesystem":731,"ArcanistLinter":1902,"ArcanistLintSeverity":1988}},"xmap":{"CheckDocLinter":["ArcanistExternalLinter"]}},"6af7410cfea496ff1d4dcc2624b6b8ea":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":781,"id":1653},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":1660,"Filesystem":1504,"ArcanistLinter":1767,"ArcanistLintSeverity":1845}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}},"6f2f22dd0f259fb2eaa284b4fab3bc29":{"have":{"class":{"PythonFormatLinter":123}},"need":{"function":{"pht":353,"id":1838},"class":{"ArcanistExternalLinter":150,"ArcanistLintMessage":1845,"Filesystem":776,"ArcanistLinter":1970,"ArcanistLintSeverity":2053}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"25781df78f6eebfb223296b8265e9d19":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2629},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2636,"Filesystem":776,"ArcanistLinter":2684,"ArcanistLintSeverity":2792}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}},"9285ad9415f8ebe564f7119e5a72c559":{"have":{"class":{"FormatStringLinter":146}},"need":{"function":{"pht":377,"csprintf":1492,"id":1872},"class":{"ArcanistExternalLinter":173,"ArcanistLintMessage":1879,"Filesystem":827,"ArcanistLinter":1956,"ArcanistLintSeverity":2044}},"xmap":{"FormatStringLinter":["ArcanistExternalLinter"]}}}
\ No newline at end of file
diff --git a/arcanist/__phutil_library_map__.php b/arcanist/__phutil_library_map__.php
index 82f69981f..cab17f1d4 100644
--- a/arcanist/__phutil_library_map__.php
+++ b/arcanist/__phutil_library_map__.php
@@ -1,26 +1,28 @@
<?php
/**
* This file is automatically generated. Use 'arc liberate' to rebuild it.
*
* @generated
* @phutil-library-version 2
*/
phutil_register_library_map(array(
'__library_version__' => 2,
'class' => array(
'AutoPEP8FormatLinter' => 'linter/AutoPEP8Linter.php',
'CheckDocLinter' => 'linter/CheckDocLinter.php',
'ClangFormatLinter' => 'linter/ClangFormatLinter.php',
+ 'FormatStringLinter' => 'linter/FormatStringLinter.php',
'PythonFormatLinter' => 'linter/PythonFormatLinter.php',
'TestsLinter' => 'linter/TestsLinter.php',
),
'function' => array(),
'xmap' => array(
'AutoPEP8FormatLinter' => 'ArcanistExternalLinter',
'CheckDocLinter' => 'ArcanistExternalLinter',
'ClangFormatLinter' => 'ArcanistExternalLinter',
+ 'FormatStringLinter' => 'ArcanistExternalLinter',
'PythonFormatLinter' => 'ArcanistExternalLinter',
'TestsLinter' => 'ArcanistExternalLinter',
),
));
diff --git a/arcanist/linter/FormatStringLinter.php b/arcanist/linter/FormatStringLinter.php
new file mode 100644
index 000000000..52d687c9b
--- /dev/null
+++ b/arcanist/linter/FormatStringLinter.php
@@ -0,0 +1,86 @@
+<?php
+
+/**
+ * Uses the lint-format-strings.py script to check the number of arguments in
+ * variadic string formatting functions.
+ */
+final class FormatStringLinter extends ArcanistExternalLinter {
+
+ public function getInfoName() {
+ return 'lint-format-strings';
+ }
+
+ public function getInfoURI() {
+ return '';
+ }
+
+ public function getInfoDescription() {
+ return pht('Check the number of arguments in variadic string formatting '.
+ 'functions');
+ }
+
+ public function getLinterName() {
+ return 'lint-format-strings';
+ }
+
+ public function getLinterConfigurationName() {
+ return 'lint-format-strings';
+ }
+
+ public function getLinterConfigurationOptions() {
+ $options = array();
+ return $options + parent::getLinterConfigurationOptions();
+ }
+
+ public function getDefaultBinary() {
+ return Filesystem::resolvePath('test/lint/lint-format-strings.py',
+ $this->getProjectRoot());
+ }
+
+ public function shouldUseInterpreter() {
+ return true;
+ }
+
+ public function getDefaultInterpreter() {
+ return "python3";
+ }
+
+ public function getInstallInstructions() {
+ return pht('The test/lint/lint-format-strings.py script is part of the '.
+ 'bitcoin-abc project');
+ }
+
+ public function shouldExpectCommandErrors() {
+ return false;
+ }
+
+ protected function getMandatoryFlags() {
+ return array();
+ }
+
+ protected function getPathArgumentForLinterFuture($path) {
+ // The path is expected to be relative to the project root
+ return csprintf('%s',
+ Filesystem::readablePath($path, $this->getProjectRoot()));
+ }
+
+ protected function parseLinterOutput($path, $err, $stdout, $stderr) {
+ $pattern = '/^(?P<file>.+): (?P<message>.+): (?P<code>.+)$/m';
+
+ $messages = array();
+
+ if (preg_match_all($pattern, $stdout, $matches, PREG_SET_ORDER)) {
+ foreach($matches as $match) {
+ $messages[] = id(new ArcanistLintMessage())
+ ->setPath($path)
+ ->setGranularity(ArcanistLinter::GRANULARITY_FILE)
+ ->setCode('STRFMT')
+ ->setSeverity(ArcanistLintSeverity::SEVERITY_ERROR)
+ ->setName('String formatting function arguments mismatch')
+ ->setDescription($match["code"]."\n".$match["message"]);
+ }
+ }
+
+ return $messages;
+ }
+}
diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py
index 550c84374..fc090b23d 100755
--- a/test/lint/lint-format-strings.py
+++ b/test/lint/lint-format-strings.py
@@ -1,293 +1,289 @@
#!/usr/bin/env python3
#
# Copyright (c) 2018 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Lint format strings: This program checks that the number of arguments passed
# to a variadic format string function matches the number of format specifiers
# in the format string.
import argparse
+import doctest
import re
import sys
FALSE_POSITIVES = [
("src/dbwrapper.cpp", "vsnprintf(p, limit - p, format, backup_ap)"),
("src/index/base.cpp", "FatalError(const char* fmt, const Args&... args)"),
("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"),
("src/util.cpp", "strprintf(_(COPYRIGHT_HOLDERS), _(COPYRIGHT_HOLDERS_SUBSTITUTION))"),
("src/util.cpp", "strprintf(COPYRIGHT_HOLDERS, COPYRIGHT_HOLDERS_SUBSTITUTION)"),
("src/seeder/main.cpp", "fprintf(stderr, help, argv[0])"),
("src/tinyformat.h", "printf(const char *fmt, const Args &... args)"),
("src/tinyformat.h", "printf(const char *fmt, TINYFORMAT_VARARGS(n))"),
]
FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS = [
("FatalError", 0),
("fprintf", 1),
("LogConnectFailure", 1),
("LogPrint", 1),
("LogPrintf", 0),
("printf", 0),
("snprintf", 2),
("sprintf", 1),
("strprintf", 0),
("vfprintf", 1),
("vprintf", 1),
("vsnprintf", 1),
("vsprintf", 1),
]
def parse_function_calls(function_name, source_code):
"""Return an array with all calls to function function_name in string source_code.
Preprocessor directives and C++ style comments ("//") in source_code are removed.
>>> len(parse_function_calls("foo", "foo();bar();foo();bar();"))
2
>>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[0].startswith("foo(1);")
True
>>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[1].startswith("foo(2);")
True
>>> len(parse_function_calls("foo", "foo();bar();// foo();bar();"))
1
>>> len(parse_function_calls("foo", "#define FOO foo();"))
0
"""
assert(type(function_name) is str and type(
source_code) is str and function_name)
lines = [re.sub("// .*", " ", line).strip()
for line in source_code.split("\n")
if not line.strip().startswith("#")]
return re.findall(r"[^a-zA-Z_](?=({}\(.*).*)".format(function_name), " " + " ".join(lines))
def normalize(s):
"""Return a normalized version of string s with newlines, tabs and C style comments ("/* ... */")
replaced with spaces. Multiple spaces are replaced with a single space.
>>> normalize(" /* nothing */ foo\tfoo /* bar */ foo ")
'foo foo foo'
"""
assert(type(s) is str)
s = s.replace("\n", " ")
s = s.replace("\t", " ")
s = re.sub("/\*.*?\*/", " ", s)
s = re.sub(" {2,}", " ", s)
return s.strip()
ESCAPE_MAP = {
r"\n": "[escaped-newline]",
r"\t": "[escaped-tab]",
r'\"': "[escaped-quote]",
}
def escape(s):
"""Return the escaped version of string s with "\\\"", "\\n" and "\\t" escaped as
"[escaped-backslash]", "[escaped-newline]" and "[escaped-tab]".
>>> unescape(escape("foo")) == "foo"
True
>>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"')
'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]'
"""
assert(type(s) is str)
for raw_value, escaped_value in ESCAPE_MAP.items():
s = s.replace(raw_value, escaped_value)
return s
def unescape(s):
"""Return the unescaped version of escaped string s.
Reverses the replacements made in function escape(s).
>>> unescape(escape("bar"))
'bar'
>>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]")
'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"'
"""
assert(type(s) is str)
for raw_value, escaped_value in ESCAPE_MAP.items():
s = s.replace(escaped_value, raw_value)
return s
def parse_function_call_and_arguments(function_name, function_call):
"""Split string function_call into an array of strings consisting of:
* the string function_call followed by "("
* the function call argument #1
* ...
* the function call argument #n
* a trailing ");"
The strings returned are in escaped form. See escape(...).
>>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");')
['foo(', '"%s",', ' "foo"', ')']
>>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");')
['foo(', '"%s",', ' "foo"', ')']
>>> parse_function_call_and_arguments("foo", 'foo("%s %s", "foo", "bar");')
['foo(', '"%s %s",', ' "foo",', ' "bar"', ')']
>>> parse_function_call_and_arguments("fooprintf", 'fooprintf("%050d", i);')
['fooprintf(', '"%050d",', ' i', ')']
>>> parse_function_call_and_arguments("foo", 'foo(bar(foobar(barfoo("foo"))), foobar); barfoo')
['foo(', 'bar(foobar(barfoo("foo"))),', ' foobar', ')']
>>> parse_function_call_and_arguments("foo", "foo()")
['foo(', '', ')']
>>> parse_function_call_and_arguments("foo", "foo(123)")
['foo(', '123', ')']
>>> parse_function_call_and_arguments("foo", 'foo("foo")')
['foo(', '"foo"', ')']
"""
assert(type(function_name) is str and type(
function_call) is str and function_name)
remaining = normalize(escape(function_call))
expected_function_call = "{}(".format(function_name)
assert(remaining.startswith(expected_function_call))
parts = [expected_function_call]
remaining = remaining[len(expected_function_call):]
open_parentheses = 1
in_string = False
parts.append("")
for char in remaining:
parts.append(parts.pop() + char)
if char == "\"":
in_string = not in_string
continue
if in_string:
continue
if char == "(":
open_parentheses += 1
continue
if char == ")":
open_parentheses -= 1
if open_parentheses > 1:
continue
if open_parentheses == 0:
parts.append(parts.pop()[:-1])
parts.append(char)
break
if char == ",":
parts.append("")
return parts
def parse_string_content(argument):
"""Return the text within quotes in string argument.
>>> parse_string_content('1 "foo %d bar" 2')
'foo %d bar'
>>> parse_string_content('1 foobar 2')
''
>>> parse_string_content('1 "bar" 2')
'bar'
>>> parse_string_content('1 "foo" 2 "bar" 3')
'foobar'
>>> parse_string_content('1 "foo" 2 " " "bar" 3')
'foo bar'
>>> parse_string_content('""')
''
>>> parse_string_content('')
''
>>> parse_string_content('1 2 3')
''
"""
assert(type(argument) is str)
string_content = ""
in_string = False
for char in normalize(escape(argument)):
if char == "\"":
in_string = not in_string
elif in_string:
string_content += char
return string_content
def count_format_specifiers(format_string):
"""Return the number of format specifiers in string format_string.
>>> count_format_specifiers("foo bar foo")
0
>>> count_format_specifiers("foo %d bar foo")
1
>>> count_format_specifiers("foo %d bar %i foo")
2
>>> count_format_specifiers("foo %d bar %i foo %% foo")
2
>>> count_format_specifiers("foo %d bar %i foo %% foo %d foo")
3
>>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo")
4
"""
assert(type(format_string) is str)
n = 0
in_specifier = False
for i, char in enumerate(format_string):
if format_string[i - 1:i + 1] == "%%" or format_string[i:i + 2] == "%%":
pass
elif char == "%":
in_specifier = True
n += 1
elif char in "aAcdeEfFgGinopsuxX":
in_specifier = False
elif in_specifier and char == "*":
n += 1
return n
def main(args_in):
""" Return a string output with information on string format errors
>>> main(["test/lint/lint-format-strings-tests.txt"])
test/lint/lint-format-strings-tests.txt: Expected 1 argument(s) after format string but found 2 argument(s): printf("%d", 1, 2)
test/lint/lint-format-strings-tests.txt: Expected 2 argument(s) after format string but found 3 argument(s): printf("%a %b", 1, 2, "anything")
test/lint/lint-format-strings-tests.txt: Expected 1 argument(s) after format string but found 0 argument(s): printf("%d")
test/lint/lint-format-strings-tests.txt: Expected 3 argument(s) after format string but found 2 argument(s): printf("%a%b%z", 1, "anything")
- 1
>>> main(["test/lint/lint-format-strings-tests-skip-arguments.txt"])
test/lint/lint-format-strings-tests-skip-arguments.txt: Expected 1 argument(s) after format string but found 2 argument(s): fprintf(skipped, "%d", 1, 2)
test/lint/lint-format-strings-tests-skip-arguments.txt: Expected 1 argument(s) after format string but found 0 argument(s): fprintf(skipped, "%d")
test/lint/lint-format-strings-tests-skip-arguments.txt: Expected 1 argument(s) after format string but found 2 argument(s): snprintf(skip1, skip2, "%d", 1, 2)
test/lint/lint-format-strings-tests-skip-arguments.txt: Expected 1 argument(s) after format string but found 0 argument(s): snprintf(skip1, skip2, "%d")
test/lint/lint-format-strings-tests-skip-arguments.txt: Could not parse function call string "snprintf(...)": snprintf(skip1, "%d")
- 1
"""
parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed "
"to a variadic format string function matches the number of format "
"specifiers in the format string.")
parser.add_argument("file", type=argparse.FileType(
"r", encoding="utf-8"), nargs="*", help="C++ source code file (e.g. foo.cpp)")
args = parser.parse_args(args_in)
- exit_code = 0
for f in args.file:
file_content = f.read()
for (function_name, skip_arguments) in FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS:
for function_call_str in parse_function_calls(function_name, file_content):
parts = parse_function_call_and_arguments(
function_name, function_call_str)
relevant_function_call_str = unescape("".join(parts))[:512]
if (f.name, relevant_function_call_str) in FALSE_POSITIVES:
continue
if len(parts) < 3 + skip_arguments:
- exit_code = 1
print("{}: Could not parse function call string \"{}(...)\": {}".format(
f.name, function_name, relevant_function_call_str))
continue
argument_count = len(parts) - 3 - skip_arguments
format_str = parse_string_content(parts[1 + skip_arguments])
format_specifier_count = count_format_specifiers(format_str)
if format_specifier_count != argument_count:
- exit_code = 1
print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(
f.name, format_specifier_count, argument_count, relevant_function_call_str))
continue
- return exit_code
if __name__ == "__main__":
- sys.exit(main(sys.argv[1:]))
+ doctest.testmod()
+ main(sys.argv[1:])
diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh
deleted file mode 100755
index 1f56f4818..000000000
--- a/test/lint/lint-format-strings.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-#
-# Lint format strings: This program checks that the number of arguments passed
-# to a variadic format string function matches the number of format specifiers
-# in the format string.
-
-export LC_ALL=C
-
-EXIT_CODE=0
-if ! python3 -m doctest test/lint/lint-format-strings.py; then
- EXIT_CODE=1
-fi
-
-test/lint/lint-format-strings.py "${1}"
-
-exit ${EXIT_CODE}