Changeset View
Changeset View
Standalone View
Standalone View
test/fuzz/test_runner.py
Show All 11 Lines | |||||
import subprocess | import subprocess | ||||
import sys | import sys | ||||
from concurrent.futures import ThreadPoolExecutor, as_completed | from concurrent.futures import ThreadPoolExecutor, as_completed | ||||
def main(): | def main(): | ||||
parser = argparse.ArgumentParser( | parser = argparse.ArgumentParser( | ||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter, | formatter_class=argparse.ArgumentDefaultsHelpFormatter, | ||||
description='''Run the fuzz targets with all inputs from the seed_dir once.''', | description="""Run the fuzz targets with all inputs from the seed_dir once.""", | ||||
) | ) | ||||
parser.add_argument( | parser.add_argument( | ||||
"-l", | "-l", | ||||
"--loglevel", | "--loglevel", | ||||
dest="loglevel", | dest="loglevel", | ||||
default="INFO", | default="INFO", | ||||
help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.", | help=( | ||||
"log events at this level and higher to the console. Can be set to DEBUG," | |||||
" INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output" | |||||
" all logs to console." | |||||
), | |||||
) | ) | ||||
parser.add_argument( | parser.add_argument( | ||||
'--valgrind', | "--valgrind", | ||||
action='store_true', | action="store_true", | ||||
help='If true, run fuzzing binaries under the valgrind memory error detector', | help="If true, run fuzzing binaries under the valgrind memory error detector", | ||||
) | ) | ||||
parser.add_argument( | parser.add_argument( | ||||
'-x', | "-x", | ||||
'--exclude', | "--exclude", | ||||
help="A comma-separated list of targets to exclude", | help="A comma-separated list of targets to exclude", | ||||
) | ) | ||||
parser.add_argument( | parser.add_argument( | ||||
'--par', | "--par", | ||||
'-j', | "-j", | ||||
type=int, | type=int, | ||||
default=4, | default=4, | ||||
help='How many targets to merge or execute in parallel.', | help="How many targets to merge or execute in parallel.", | ||||
) | ) | ||||
parser.add_argument( | parser.add_argument( | ||||
'seed_dir', | "seed_dir", | ||||
help='The seed corpus to run on (must contain subfolders for each fuzz target).', | help=( | ||||
"The seed corpus to run on (must contain subfolders for each fuzz target)." | |||||
), | |||||
) | ) | ||||
parser.add_argument( | parser.add_argument( | ||||
'target', | "target", | ||||
nargs='*', | nargs="*", | ||||
help='The target(s) to run. Default is to run all targets.', | help="The target(s) to run. Default is to run all targets.", | ||||
) | ) | ||||
parser.add_argument( | parser.add_argument( | ||||
'--m_dir', | "--m_dir", | ||||
help='Merge inputs from this directory into the seed_dir. Needs /target subdirectory.', | help=( | ||||
"Merge inputs from this directory into the seed_dir. Needs /target" | |||||
" subdirectory." | |||||
), | |||||
) | ) | ||||
parser.add_argument( | parser.add_argument( | ||||
'-g', | "-g", | ||||
'--generate', | "--generate", | ||||
action='store_true', | action="store_true", | ||||
help='Create new corpus seeds (or extend the existing ones) by running' | help=( | ||||
' the given targets for a finite number of times. Outputs them to' | "Create new corpus seeds (or extend the existing ones) by running" | ||||
' the passed seed_dir.' | " the given targets for a finite number of times. Outputs them to" | ||||
" the passed seed_dir." | |||||
), | |||||
) | ) | ||||
args = parser.parse_args() | args = parser.parse_args() | ||||
# Set up logging | # Set up logging | ||||
logging.basicConfig( | logging.basicConfig( | ||||
format='%(message)s', | format="%(message)s", | ||||
level=int(args.loglevel) if args.loglevel.isdigit( | level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(), | ||||
) else args.loglevel.upper(), | |||||
) | ) | ||||
# Read config generated by configure. | # Read config generated by configure. | ||||
config = configparser.ConfigParser() | config = configparser.ConfigParser() | ||||
configfile = f"{os.path.abspath(os.path.dirname(__file__))}/../config.ini" | configfile = f"{os.path.abspath(os.path.dirname(__file__))}/../config.ini" | ||||
config.read_file(open(configfile, encoding="utf8")) | config.read_file(open(configfile, encoding="utf8")) | ||||
if not config["components"].getboolean("ENABLE_FUZZ"): | if not config["components"].getboolean("ENABLE_FUZZ"): | ||||
logging.error("Must have fuzz targets built") | logging.error("Must have fuzz targets built") | ||||
sys.exit(1) | sys.exit(1) | ||||
test_dir = os.path.join( | test_dir = os.path.join(config["environment"]["BUILDDIR"], "src", "test", "fuzz") | ||||
config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz') | |||||
# Build list of tests | # Build list of tests | ||||
test_list_all = [ | test_list_all = [ | ||||
f for f in os.listdir(test_dir) | f | ||||
if os.path.isfile(os.path.join(test_dir, f)) and | for f in os.listdir(test_dir) | ||||
os.access(os.path.join(test_dir, f), os.X_OK)] | if os.path.isfile(os.path.join(test_dir, f)) | ||||
and os.access(os.path.join(test_dir, f), os.X_OK) | |||||
] | |||||
if not test_list_all: | if not test_list_all: | ||||
logging.error("No fuzz targets found") | logging.error("No fuzz targets found") | ||||
sys.exit(1) | sys.exit(1) | ||||
logging.debug( | logging.debug( | ||||
f"{len(test_list_all)} fuzz target(s) found: {' '.join(sorted(test_list_all))}") | f"{len(test_list_all)} fuzz target(s) found: {' '.join(sorted(test_list_all))}" | ||||
) | |||||
# By default run all | # By default run all | ||||
args.target = args.target or test_list_all | args.target = args.target or test_list_all | ||||
test_list_error = list(set(args.target).difference(set(test_list_all))) | test_list_error = list(set(args.target).difference(set(test_list_all))) | ||||
if test_list_error: | if test_list_error: | ||||
logging.error( | logging.error(f"Unknown fuzz targets selected: {test_list_error}") | ||||
f"Unknown fuzz targets selected: {test_list_error}") | test_list_selection = list(set(test_list_all).intersection(set(args.target))) | ||||
test_list_selection = list( | |||||
set(test_list_all).intersection(set(args.target))) | |||||
if not test_list_selection: | if not test_list_selection: | ||||
logging.error("No fuzz targets selected") | logging.error("No fuzz targets selected") | ||||
if args.exclude: | if args.exclude: | ||||
for excluded_target in args.exclude.split(","): | for excluded_target in args.exclude.split(","): | ||||
if excluded_target not in test_list_selection: | if excluded_target not in test_list_selection: | ||||
logging.error( | logging.error( | ||||
f"Target \"{excluded_target}\" not found in current target list.") | f'Target "{excluded_target}" not found in current target list.' | ||||
) | |||||
continue | continue | ||||
test_list_selection.remove(excluded_target) | test_list_selection.remove(excluded_target) | ||||
test_list_selection.sort() | test_list_selection.sort() | ||||
logging.info( | logging.info( | ||||
"{} of {} detected fuzz target(s) selected: {}".format( | "{} of {} detected fuzz target(s) selected: {}".format( | ||||
len(test_list_selection), | len(test_list_selection), len(test_list_all), " ".join(test_list_selection) | ||||
len(test_list_all), | ) | ||||
" ".join(test_list_selection))) | ) | ||||
if not args.generate: | if not args.generate: | ||||
test_list_seedless = [] | test_list_seedless = [] | ||||
for t in test_list_selection: | for t in test_list_selection: | ||||
corpus_path = os.path.join(args.seed_dir, t) | corpus_path = os.path.join(args.seed_dir, t) | ||||
if not os.path.exists(corpus_path) or len( | if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0: | ||||
os.listdir(corpus_path)) == 0: | |||||
test_list_seedless.append(t) | test_list_seedless.append(t) | ||||
test_list_seedless.sort() | test_list_seedless.sort() | ||||
if test_list_seedless: | if test_list_seedless: | ||||
logging.info( | logging.info( | ||||
"Fuzzing harnesses lacking a seed corpus: {}".format( | "Fuzzing harnesses lacking a seed corpus: {}".format( | ||||
" ".join(test_list_seedless) | " ".join(test_list_seedless) | ||||
) | ) | ||||
) | ) | ||||
logging.info( | logging.info( | ||||
"Please consider adding a fuzz seed corpus at https://github.com/Bitcoin-ABC/qa-assets") | "Please consider adding a fuzz seed corpus at" | ||||
" https://github.com/Bitcoin-ABC/qa-assets" | |||||
) | |||||
try: | try: | ||||
help_output = subprocess.run( | help_output = subprocess.run( | ||||
args=[ | args=[ | ||||
os.path.join(test_dir, test_list_selection[0]), | os.path.join(test_dir, test_list_selection[0]), | ||||
'-help=1', | "-help=1", | ||||
], | ], | ||||
timeout=20, | timeout=20, | ||||
check=True, | check=True, | ||||
stderr=subprocess.PIPE, | stderr=subprocess.PIPE, | ||||
universal_newlines=True, | universal_newlines=True, | ||||
).stderr | ).stderr | ||||
if "libFuzzer" not in help_output: | if "libFuzzer" not in help_output: | ||||
logging.error("Must be built with libFuzzer") | logging.error("Must be built with libFuzzer") | ||||
sys.exit(1) | sys.exit(1) | ||||
except subprocess.TimeoutExpired: | except subprocess.TimeoutExpired: | ||||
logging.error( | logging.error("subprocess timed out: Currently only libFuzzer is supported") | ||||
"subprocess timed out: Currently only libFuzzer is supported") | |||||
sys.exit(1) | sys.exit(1) | ||||
with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool: | with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool: | ||||
if args.generate: | if args.generate: | ||||
return generate_corpus_seeds( | return generate_corpus_seeds( | ||||
fuzz_pool=fuzz_pool, | fuzz_pool=fuzz_pool, | ||||
test_dir=test_dir, | test_dir=test_dir, | ||||
seed_dir=args.seed_dir, | seed_dir=args.seed_dir, | ||||
Show All 24 Lines | def generate_corpus_seeds(*, fuzz_pool, test_dir, seed_dir, targets): | ||||
Run {targets} without input, and outputs the generated corpus seeds to | Run {targets} without input, and outputs the generated corpus seeds to | ||||
{seed_dir}. | {seed_dir}. | ||||
""" | """ | ||||
logging.info(f"Generating corpus seeds to {seed_dir}") | logging.info(f"Generating corpus seeds to {seed_dir}") | ||||
def job(command): | def job(command): | ||||
logging.debug(f"Running '{' '.join(command)}'\n") | logging.debug(f"Running '{' '.join(command)}'\n") | ||||
logging.debug("Command '{}' output:\n'{}'\n".format( | logging.debug( | ||||
' '.join(command), | "Command '{}' output:\n'{}'\n".format( | ||||
subprocess.run(command, check=True, stderr=subprocess.PIPE, | " ".join(command), | ||||
universal_newlines=True).stderr | subprocess.run( | ||||
)) | command, check=True, stderr=subprocess.PIPE, universal_newlines=True | ||||
).stderr, | |||||
) | |||||
) | |||||
futures = [] | futures = [] | ||||
for target in targets: | for target in targets: | ||||
target_seed_dir = os.path.join(seed_dir, target) | target_seed_dir = os.path.join(seed_dir, target) | ||||
os.makedirs(target_seed_dir, exist_ok=True) | os.makedirs(target_seed_dir, exist_ok=True) | ||||
command = [ | command = [ | ||||
os.path.join(test_dir, target), | os.path.join(test_dir, target), | ||||
"-runs=100000", | "-runs=100000", | ||||
target_seed_dir, | target_seed_dir, | ||||
] | ] | ||||
futures.append(fuzz_pool.submit(job, command)) | futures.append(fuzz_pool.submit(job, command)) | ||||
for future in as_completed(futures): | for future in as_completed(futures): | ||||
future.result() | future.result() | ||||
def merge_inputs(*, fuzz_pool, corpus, test_list, test_dir, merge_dir): | def merge_inputs(*, fuzz_pool, corpus, test_list, test_dir, merge_dir): | ||||
logging.info( | logging.info( | ||||
f"Merge the inputs in the passed dir into the seed_dir. Passed dir {merge_dir}") | f"Merge the inputs in the passed dir into the seed_dir. Passed dir {merge_dir}" | ||||
) | |||||
jobs = [] | jobs = [] | ||||
for t in test_list: | for t in test_list: | ||||
args = [ | args = [ | ||||
os.path.join(test_dir, t), | os.path.join(test_dir, t), | ||||
'-merge=1', | "-merge=1", | ||||
# Also done by oss-fuzz | # Also done by oss-fuzz | ||||
# https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487 | # https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487 | ||||
'-use_value_profile=1', | "-use_value_profile=1", | ||||
os.path.join(corpus, t), | os.path.join(corpus, t), | ||||
os.path.join(merge_dir, t), | os.path.join(merge_dir, t), | ||||
] | ] | ||||
os.makedirs(os.path.join(corpus, t), exist_ok=True) | os.makedirs(os.path.join(corpus, t), exist_ok=True) | ||||
os.makedirs(os.path.join(merge_dir, t), exist_ok=True) | os.makedirs(os.path.join(merge_dir, t), exist_ok=True) | ||||
def job(t, args): | def job(t, args): | ||||
output = f"Run {t} with args {' '.join(args)}\n" | output = f"Run {t} with args {' '.join(args)}\n" | ||||
output += subprocess.run(args, | output += subprocess.run( | ||||
check=True, | args, check=True, stderr=subprocess.PIPE, universal_newlines=True | ||||
stderr=subprocess.PIPE, | ).stderr | ||||
universal_newlines=True).stderr | |||||
logging.debug(output) | logging.debug(output) | ||||
jobs.append(fuzz_pool.submit(job, t, args)) | jobs.append(fuzz_pool.submit(job, t, args)) | ||||
for future in as_completed(jobs): | for future in as_completed(jobs): | ||||
future.result() | future.result() | ||||
def run_once(*, fuzz_pool, corpus, test_list, test_dir, use_valgrind): | def run_once(*, fuzz_pool, corpus, test_list, test_dir, use_valgrind): | ||||
jobs = [] | jobs = [] | ||||
for t in test_list: | for t in test_list: | ||||
corpus_path = os.path.join(corpus, t) | corpus_path = os.path.join(corpus, t) | ||||
os.makedirs(corpus_path, exist_ok=True) | os.makedirs(corpus_path, exist_ok=True) | ||||
args = [ | args = [ | ||||
os.path.join(test_dir, t), | os.path.join(test_dir, t), | ||||
'-runs=1', | "-runs=1", | ||||
corpus_path, | corpus_path, | ||||
] | ] | ||||
if use_valgrind: | if use_valgrind: | ||||
args = [ | args = ["valgrind", "--quiet", "--error-exitcode=1"] + args | ||||
'valgrind', | |||||
'--quiet', | |||||
'--error-exitcode=1'] + args | |||||
def job(t, args): | def job(t, args): | ||||
output = f'Run {t} with args {args}' | output = f"Run {t} with args {args}" | ||||
result = subprocess.run( | result = subprocess.run( | ||||
args, | args, stderr=subprocess.PIPE, universal_newlines=True | ||||
stderr=subprocess.PIPE, | ) | ||||
universal_newlines=True) | |||||
output += result.stderr | output += result.stderr | ||||
return output, result | return output, result | ||||
jobs.append(fuzz_pool.submit(job, t, args)) | jobs.append(fuzz_pool.submit(job, t, args)) | ||||
for future in as_completed(jobs): | for future in as_completed(jobs): | ||||
output, result = future.result() | output, result = future.result() | ||||
logging.debug(output) | logging.debug(output) | ||||
try: | try: | ||||
result.check_returncode() | result.check_returncode() | ||||
except subprocess.CalledProcessError as e: | except subprocess.CalledProcessError as e: | ||||
if e.stdout: | if e.stdout: | ||||
logging.info(e.stdout) | logging.info(e.stdout) | ||||
if e.stderr: | if e.stderr: | ||||
logging.info(e.stderr) | logging.info(e.stderr) | ||||
logging.info( | logging.info( | ||||
"Target \"{}\" failed with exit code {}".format( | 'Target "{}" failed with exit code {}'.format( | ||||
" ".join( | " ".join(result.args), e.returncode | ||||
result.args), | ) | ||||
e.returncode)) | ) | ||||
sys.exit(1) | sys.exit(1) | ||||
if __name__ == '__main__': | if __name__ == "__main__": | ||||
main() | main() |