Differential D13889 Diff 40286 test/fuzz/test_runner.py

Changeset View

Standalone View

test/fuzz/test_runner.py

Show All 11 Lines
import subprocess		import subprocess
import sys		import sys
from concurrent.futures import ThreadPoolExecutor, as_completed		from concurrent.futures import ThreadPoolExecutor, as_completed


def main():		def main():
parser = argparse.ArgumentParser(		parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,		formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='''Run the fuzz targets with all inputs from the seed_dir once.''',		description="""Run the fuzz targets with all inputs from the seed_dir once.""",
)		)
parser.add_argument(		parser.add_argument(
"-l",		"-l",
"--loglevel",		"--loglevel",
dest="loglevel",		dest="loglevel",
default="INFO",		default="INFO",
help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",		help=(
		"log events at this level and higher to the console. Can be set to DEBUG,"
		" INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output"
		" all logs to console."
		),
)		)
parser.add_argument(		parser.add_argument(
'--valgrind',		"--valgrind",
action='store_true',		action="store_true",
help='If true, run fuzzing binaries under the valgrind memory error detector',		help="If true, run fuzzing binaries under the valgrind memory error detector",
)		)
parser.add_argument(		parser.add_argument(
'-x',		"-x",
'--exclude',		"--exclude",
help="A comma-separated list of targets to exclude",		help="A comma-separated list of targets to exclude",
)		)
parser.add_argument(		parser.add_argument(
'--par',		"--par",
'-j',		"-j",
type=int,		type=int,
default=4,		default=4,
help='How many targets to merge or execute in parallel.',		help="How many targets to merge or execute in parallel.",
)		)
parser.add_argument(		parser.add_argument(
'seed_dir',		"seed_dir",
help='The seed corpus to run on (must contain subfolders for each fuzz target).',		help=(
		"The seed corpus to run on (must contain subfolders for each fuzz target)."
		),
)		)
parser.add_argument(		parser.add_argument(
'target',		"target",
nargs='*',		nargs="*",
help='The target(s) to run. Default is to run all targets.',		help="The target(s) to run. Default is to run all targets.",
)		)
parser.add_argument(		parser.add_argument(
'--m_dir',		"--m_dir",
help='Merge inputs from this directory into the seed_dir. Needs /target subdirectory.',		help=(
		"Merge inputs from this directory into the seed_dir. Needs /target"
		" subdirectory."
		),
)		)
parser.add_argument(		parser.add_argument(
'-g',		"-g",
'--generate',		"--generate",
action='store_true',		action="store_true",
help='Create new corpus seeds (or extend the existing ones) by running'		help=(
' the given targets for a finite number of times. Outputs them to'		"Create new corpus seeds (or extend the existing ones) by running"
' the passed seed_dir.'		" the given targets for a finite number of times. Outputs them to"
		" the passed seed_dir."
		),
)		)

args = parser.parse_args()		args = parser.parse_args()

# Set up logging		# Set up logging
logging.basicConfig(		logging.basicConfig(
format='%(message)s',		format="%(message)s",
level=int(args.loglevel) if args.loglevel.isdigit(		level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
) else args.loglevel.upper(),
)		)

# Read config generated by configure.		# Read config generated by configure.
config = configparser.ConfigParser()		config = configparser.ConfigParser()
configfile = f"{os.path.abspath(os.path.dirname(__file__))}/../config.ini"		configfile = f"{os.path.abspath(os.path.dirname(__file__))}/../config.ini"
config.read_file(open(configfile, encoding="utf8"))		config.read_file(open(configfile, encoding="utf8"))

if not config["components"].getboolean("ENABLE_FUZZ"):		if not config["components"].getboolean("ENABLE_FUZZ"):
logging.error("Must have fuzz targets built")		logging.error("Must have fuzz targets built")
sys.exit(1)		sys.exit(1)
test_dir = os.path.join(		test_dir = os.path.join(config["environment"]["BUILDDIR"], "src", "test", "fuzz")
config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz')

# Build list of tests		# Build list of tests
test_list_all = [		test_list_all = [
f for f in os.listdir(test_dir)		f
if os.path.isfile(os.path.join(test_dir, f)) and		for f in os.listdir(test_dir)
os.access(os.path.join(test_dir, f), os.X_OK)]		if os.path.isfile(os.path.join(test_dir, f))
		and os.access(os.path.join(test_dir, f), os.X_OK)
		]

if not test_list_all:		if not test_list_all:
logging.error("No fuzz targets found")		logging.error("No fuzz targets found")
sys.exit(1)		sys.exit(1)

logging.debug(		logging.debug(
f"{len(test_list_all)} fuzz target(s) found: {' '.join(sorted(test_list_all))}")		f"{len(test_list_all)} fuzz target(s) found: {' '.join(sorted(test_list_all))}"
		)

# By default run all		# By default run all
args.target = args.target or test_list_all		args.target = args.target or test_list_all
test_list_error = list(set(args.target).difference(set(test_list_all)))		test_list_error = list(set(args.target).difference(set(test_list_all)))
if test_list_error:		if test_list_error:
logging.error(		logging.error(f"Unknown fuzz targets selected: {test_list_error}")
f"Unknown fuzz targets selected: {test_list_error}")		test_list_selection = list(set(test_list_all).intersection(set(args.target)))
test_list_selection = list(
set(test_list_all).intersection(set(args.target)))
if not test_list_selection:		if not test_list_selection:
logging.error("No fuzz targets selected")		logging.error("No fuzz targets selected")
if args.exclude:		if args.exclude:
for excluded_target in args.exclude.split(","):		for excluded_target in args.exclude.split(","):
if excluded_target not in test_list_selection:		if excluded_target not in test_list_selection:
logging.error(		logging.error(
f"Target \"{excluded_target}\" not found in current target list.")		f'Target "{excluded_target}" not found in current target list.'
		)
continue		continue
test_list_selection.remove(excluded_target)		test_list_selection.remove(excluded_target)
test_list_selection.sort()		test_list_selection.sort()

logging.info(		logging.info(
"{} of {} detected fuzz target(s) selected: {}".format(		"{} of {} detected fuzz target(s) selected: {}".format(
len(test_list_selection),		len(test_list_selection), len(test_list_all), " ".join(test_list_selection)
len(test_list_all),		)
" ".join(test_list_selection)))		)

if not args.generate:		if not args.generate:
test_list_seedless = []		test_list_seedless = []
for t in test_list_selection:		for t in test_list_selection:
corpus_path = os.path.join(args.seed_dir, t)		corpus_path = os.path.join(args.seed_dir, t)
if not os.path.exists(corpus_path) or len(		if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0:
os.listdir(corpus_path)) == 0:
test_list_seedless.append(t)		test_list_seedless.append(t)
test_list_seedless.sort()		test_list_seedless.sort()
if test_list_seedless:		if test_list_seedless:
logging.info(		logging.info(
"Fuzzing harnesses lacking a seed corpus: {}".format(		"Fuzzing harnesses lacking a seed corpus: {}".format(
" ".join(test_list_seedless)		" ".join(test_list_seedless)
)		)
)		)
logging.info(		logging.info(
"Please consider adding a fuzz seed corpus at https://github.com/Bitcoin-ABC/qa-assets")		"Please consider adding a fuzz seed corpus at"
		" https://github.com/Bitcoin-ABC/qa-assets"
		)

try:		try:
help_output = subprocess.run(		help_output = subprocess.run(
args=[		args=[
os.path.join(test_dir, test_list_selection[0]),		os.path.join(test_dir, test_list_selection[0]),
'-help=1',		"-help=1",
],		],
timeout=20,		timeout=20,
check=True,		check=True,
stderr=subprocess.PIPE,		stderr=subprocess.PIPE,
universal_newlines=True,		universal_newlines=True,
).stderr		).stderr
if "libFuzzer" not in help_output:		if "libFuzzer" not in help_output:
logging.error("Must be built with libFuzzer")		logging.error("Must be built with libFuzzer")
sys.exit(1)		sys.exit(1)
except subprocess.TimeoutExpired:		except subprocess.TimeoutExpired:
logging.error(		logging.error("subprocess timed out: Currently only libFuzzer is supported")
"subprocess timed out: Currently only libFuzzer is supported")
sys.exit(1)		sys.exit(1)

with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool:		with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool:
if args.generate:		if args.generate:
return generate_corpus_seeds(		return generate_corpus_seeds(
fuzz_pool=fuzz_pool,		fuzz_pool=fuzz_pool,
test_dir=test_dir,		test_dir=test_dir,
seed_dir=args.seed_dir,		seed_dir=args.seed_dir,
Show All 24 Lines	def generate_corpus_seeds(*, fuzz_pool, test_dir, seed_dir, targets):

Run {targets} without input, and outputs the generated corpus seeds to		Run {targets} without input, and outputs the generated corpus seeds to
{seed_dir}.		{seed_dir}.
"""		"""
logging.info(f"Generating corpus seeds to {seed_dir}")		logging.info(f"Generating corpus seeds to {seed_dir}")

def job(command):		def job(command):
logging.debug(f"Running '{' '.join(command)}'\n")		logging.debug(f"Running '{' '.join(command)}'\n")
logging.debug("Command '{}' output:\n'{}'\n".format(		logging.debug(
' '.join(command),		"Command '{}' output:\n'{}'\n".format(
subprocess.run(command, check=True, stderr=subprocess.PIPE,		" ".join(command),
universal_newlines=True).stderr		subprocess.run(
))		command, check=True, stderr=subprocess.PIPE, universal_newlines=True
		).stderr,
		)
		)

futures = []		futures = []
for target in targets:		for target in targets:
target_seed_dir = os.path.join(seed_dir, target)		target_seed_dir = os.path.join(seed_dir, target)
os.makedirs(target_seed_dir, exist_ok=True)		os.makedirs(target_seed_dir, exist_ok=True)
command = [		command = [
os.path.join(test_dir, target),		os.path.join(test_dir, target),
"-runs=100000",		"-runs=100000",
target_seed_dir,		target_seed_dir,
]		]
futures.append(fuzz_pool.submit(job, command))		futures.append(fuzz_pool.submit(job, command))

for future in as_completed(futures):		for future in as_completed(futures):
future.result()		future.result()


def merge_inputs(*, fuzz_pool, corpus, test_list, test_dir, merge_dir):		def merge_inputs(*, fuzz_pool, corpus, test_list, test_dir, merge_dir):
logging.info(		logging.info(
f"Merge the inputs in the passed dir into the seed_dir. Passed dir {merge_dir}")		f"Merge the inputs in the passed dir into the seed_dir. Passed dir {merge_dir}"
		)
jobs = []		jobs = []
for t in test_list:		for t in test_list:
args = [		args = [
os.path.join(test_dir, t),		os.path.join(test_dir, t),
'-merge=1',		"-merge=1",
# Also done by oss-fuzz		# Also done by oss-fuzz
# https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487		# https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487
'-use_value_profile=1',		"-use_value_profile=1",
os.path.join(corpus, t),		os.path.join(corpus, t),
os.path.join(merge_dir, t),		os.path.join(merge_dir, t),
]		]
os.makedirs(os.path.join(corpus, t), exist_ok=True)		os.makedirs(os.path.join(corpus, t), exist_ok=True)
os.makedirs(os.path.join(merge_dir, t), exist_ok=True)		os.makedirs(os.path.join(merge_dir, t), exist_ok=True)

def job(t, args):		def job(t, args):
output = f"Run {t} with args {' '.join(args)}\n"		output = f"Run {t} with args {' '.join(args)}\n"
output += subprocess.run(args,		output += subprocess.run(
check=True,		args, check=True, stderr=subprocess.PIPE, universal_newlines=True
stderr=subprocess.PIPE,		).stderr
universal_newlines=True).stderr
logging.debug(output)		logging.debug(output)

jobs.append(fuzz_pool.submit(job, t, args))		jobs.append(fuzz_pool.submit(job, t, args))

for future in as_completed(jobs):		for future in as_completed(jobs):
future.result()		future.result()


def run_once(*, fuzz_pool, corpus, test_list, test_dir, use_valgrind):		def run_once(*, fuzz_pool, corpus, test_list, test_dir, use_valgrind):
jobs = []		jobs = []
for t in test_list:		for t in test_list:
corpus_path = os.path.join(corpus, t)		corpus_path = os.path.join(corpus, t)
os.makedirs(corpus_path, exist_ok=True)		os.makedirs(corpus_path, exist_ok=True)
args = [		args = [
os.path.join(test_dir, t),		os.path.join(test_dir, t),
'-runs=1',		"-runs=1",
corpus_path,		corpus_path,
]		]
if use_valgrind:		if use_valgrind:
args = [		args = ["valgrind", "--quiet", "--error-exitcode=1"] + args
'valgrind',
'--quiet',
'--error-exitcode=1'] + args

def job(t, args):		def job(t, args):
output = f'Run {t} with args {args}'		output = f"Run {t} with args {args}"
result = subprocess.run(		result = subprocess.run(
args,		args, stderr=subprocess.PIPE, universal_newlines=True
stderr=subprocess.PIPE,		)
universal_newlines=True)
output += result.stderr		output += result.stderr
return output, result		return output, result

jobs.append(fuzz_pool.submit(job, t, args))		jobs.append(fuzz_pool.submit(job, t, args))

for future in as_completed(jobs):		for future in as_completed(jobs):
output, result = future.result()		output, result = future.result()
logging.debug(output)		logging.debug(output)
try:		try:
result.check_returncode()		result.check_returncode()
except subprocess.CalledProcessError as e:		except subprocess.CalledProcessError as e:
if e.stdout:		if e.stdout:
logging.info(e.stdout)		logging.info(e.stdout)
if e.stderr:		if e.stderr:
logging.info(e.stderr)		logging.info(e.stderr)
logging.info(		logging.info(
"Target \"{}\" failed with exit code {}".format(		'Target "{}" failed with exit code {}'.format(
" ".join(		" ".join(result.args), e.returncode
result.args),		)
e.returncode))		)
sys.exit(1)		sys.exit(1)


if __name__ == '__main__':		if __name__ == "__main__":
main()		main()