diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index c2bdd77bb..1496d65a1 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -1,191 +1,194 @@ #!/usr/bin/env python3 # Copyright (c) 2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run fuzz test targets. """ import argparse import configparser import logging import os import subprocess import sys def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='''Run the fuzz targets with all inputs from the seed_dir once.''', ) parser.add_argument( "-l", "--loglevel", dest="loglevel", default="INFO", help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.", ) parser.add_argument( '--valgrind', action='store_true', help='If true, run fuzzing binaries under the valgrind memory error detector. Valgrind 3.14 or later required.', ) parser.add_argument( 'seed_dir', help='The seed corpus to run on (must contain subfolders for each fuzz target).', ) parser.add_argument( 'target', nargs='*', help='The target(s) to run. Default is to run all targets.', ) parser.add_argument( '--m_dir', help='Merge inputs from this directory into the seed_dir. Needs /target subdirectory.', ) args = parser.parse_args() # Set up logging logging.basicConfig( format='%(message)s', level=int(args.loglevel) if args.loglevel.isdigit( ) else args.loglevel.upper(), ) # Read config generated by configure. config = configparser.ConfigParser() configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" config.read_file(open(configfile, encoding="utf8")) if not config["components"].getboolean("ENABLE_FUZZ"): logging.error("Must have fuzz targets built") sys.exit(1) test_dir = os.path.join( config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz') # Build list of tests test_list_all = [ f for f in os.listdir(test_dir) if os.path.isfile(os.path.join(test_dir, f)) and os.access(os.path.join(test_dir, f), os.X_OK)] if not test_list_all: logging.error("No fuzz targets found") sys.exit(1) logging.info("Fuzz targets found: {}".format(test_list_all)) # By default run all args.target = args.target or test_list_all test_list_error = list(set(args.target).difference(set(test_list_all))) if test_list_error: logging.error( "Unknown fuzz targets selected: {}".format(test_list_error)) test_list_selection = list( set(test_list_all).intersection(set(args.target))) if not test_list_selection: logging.error("No fuzz targets selected") logging.info("Fuzz targets selected: {}".format(test_list_selection)) test_list_seedless = [] for t in test_list_selection: corpus_path = os.path.join(args.seed_dir, t) if not os.path.exists(corpus_path) or len( os.listdir(corpus_path)) == 0: test_list_seedless.append(t) test_list_seedless.sort() if test_list_seedless: logging.info( "Fuzzing harnesses lacking a seed corpus: {}".format( " ".join(test_list_seedless) ) ) logging.info( "Please consider adding a fuzz seed corpus at https://github.com/Bitcoin-ABC/qa-assets") try: help_output = subprocess.run( args=[ os.path.join(test_dir, test_list_selection[0]), '-help=1', ], timeout=20, check=True, stderr=subprocess.PIPE, universal_newlines=True, ).stderr if "libFuzzer" not in help_output: logging.error("Must be built with libFuzzer") sys.exit(1) except subprocess.TimeoutExpired: logging.error( "subprocess timed out: Currently only libFuzzer is supported") sys.exit(1) if args.m_dir: merge_inputs( corpus=args.seed_dir, test_list=test_list_selection, build_dir=config["environment"]["BUILDDIR"], merge_dir=args.m_dir, ) return run_once( corpus=args.seed_dir, test_list=test_list_selection, test_dir=test_dir, use_valgrind=args.valgrind, ) def merge_inputs(*, corpus, test_list, build_dir, merge_dir): logging.info( "Merge the inputs in the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) for t in test_list: args = [ os.path.join(build_dir, 'src', 'test', 'fuzz', t), '-merge=1', + # Also done by oss-fuzz + # https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487 + '-use_value_profile=1', os.path.join(corpus, t), os.path.join(merge_dir, t), ] os.makedirs(os.path.join(corpus, t), exist_ok=True) os.makedirs(os.path.join(merge_dir, t), exist_ok=True) logging.debug('Run {} with args {}'.format(t, args)) output = subprocess.run( args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr logging.debug('Output: {}'.format(output)) def run_once(*, corpus, test_list, test_dir, use_valgrind): for t in test_list: corpus_path = os.path.join(corpus, t) os.makedirs(corpus_path, exist_ok=True) args = [ os.path.join(test_dir, t), '-runs=1', '-detect_leaks=0', corpus_path, ] if use_valgrind: args = [ 'valgrind', '--quiet', '--error-exitcode=1', '--exit-on-first-error=yes'] + args logging.debug('Run {} with args {}'.format(t, args)) result = subprocess.run( args, stderr=subprocess.PIPE, universal_newlines=True) output = result.stderr logging.debug('Output: {}'.format(output)) result.check_returncode() if __name__ == '__main__': main()