diff --git a/doc/fuzzing.md b/doc/fuzzing.md index d66b82b8b..816535de8 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -1,105 +1,100 @@ Fuzz-testing Bitcoin ABC ========================== A special test harness in `src/test/fuzz/` is provided for each fuzz target to provide an easy entry point for fuzzers and the like. In this document we'll describe how to use it with AFL and libFuzzer. ## Preparing fuzzing AFL needs an input directory with examples, and an output directory where it will place examples that it found. These can be anywhere in the file system, we'll define environment variables to make it easy to reference them. libFuzzer will use the input directory as output directory. Extract the example seeds (or other starting inputs) into the inputs directory before starting fuzzing. ``` git clone https://github.com/Bitcoin-ABC/qa-assets export DIR_FUZZ_IN=$PWD/qa-assets/fuzz_seed_corpus ``` Only for AFL: ``` mkdir outputs export AFLOUT=$PWD/outputs ``` ## AFL ### Building AFL It is recommended to always use the latest version of afl: ``` wget http://lcamtuf.coredump.cx/afl/releases/afl-latest.tgz tar -zxvf afl-latest.tgz cd afl- make export AFLPATH=$PWD ``` ### Instrumentation To build Bitcoin ABC using AFL instrumentation (this assumes that the `AFLPATH` was set as above): ``` mkdir -p buildFuzzer cd buildFuzzer -cmake -GNinja .. -DCCACHE=OFF -DCMAKE_C_COMPILER=afl-gcc -DCMAKE_CXX_COMPILER=afl-g++ +cmake -GNinja .. -DCMAKE_C_COMPILER=afl-gcc -DCMAKE_CXX_COMPILER=afl-g++ export AFL_HARDEN=1 ninja bitcoin-fuzzers ``` -We disable ccache because we don't want to pollute the ccache with instrumented -objects, and similarly don't want to use non-instrumented cached objects linked -in. - The fuzzing can be sped up significantly (~200x) by using `afl-clang-fast` and `afl-clang-fast++` in place of `afl-gcc` and `afl-g++` when compiling. When compiling using `afl-clang-fast`/`afl-clang-fast++` the resulting binary will be instrumented in such a way that the AFL features "persistent mode" and "deferred forkserver" can be used. See https://github.com/mcarpenter/afl/tree/master/llvm_mode for details. ### Fuzzing To start the actual fuzzing use: ``` export FUZZ_TARGET=fuzz_target_foo # Pick a fuzz_target mkdir ${AFLOUT}/${FUZZ_TARGET} $AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- src/test/fuzz/${FUZZ_TARGET} ``` You may have to change a few kernel parameters to test optimally - `afl-fuzz` will print an error and suggestion if so. ## libFuzzer A recent version of `clang`, the address/undefined sanitizers (ASan/UBSan) and libFuzzer is needed (all found in the `compiler-rt` runtime libraries package). To build all fuzz targets with libFuzzer, run ``` mkdir -p buildFuzzer cd buildFuzzer cmake -GNinja .. \ - -DCCACHE=OFF \ -DCMAKE_C_COMPILER=clang \ -DCMAKE_CXX_COMPILER=clang++ \ -DENABLE_SANITIZERS="fuzzer;address;undefined" ninja bitcoin-fuzzers ``` The fuzzer needs some inputs to work on, but the inputs or seeds can be used interchangeably between libFuzzer and AFL. See https://llvm.org/docs/LibFuzzer.html#running on how to run the libFuzzer instrumented executable. Alternatively run the script in `./test/fuzz/test_runner.py` and provide it with the `${DIR_FUZZ_IN}` created earlier. diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index a8cb64a05..ec422ebf7 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -1,168 +1,190 @@ #!/usr/bin/env python3 # Copyright (c) 2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run fuzz test targets. """ import argparse import configparser import logging import os import subprocess import sys def main(): parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description='''Run the fuzz targets with all inputs from the seed_dir once.''', + ) parser.add_argument( "-l", "--loglevel", dest="loglevel", default="INFO", help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.", ) - parser.add_argument( - '--export_coverage', - action='store_true', - help='If true, export coverage information to files in the seed corpus', - ) parser.add_argument( '--valgrind', action='store_true', help='If true, run fuzzing binaries under the valgrind memory error detector. Valgrind 3.14 or later required.', ) parser.add_argument( 'seed_dir', help='The seed corpus to run on (must contain subfolders for each fuzz target).', ) parser.add_argument( 'target', nargs='*', help='The target(s) to run. Default is to run all targets.', ) + parser.add_argument( + '--m_dir', + help='Merge inputs from this directory into the seed_dir. Needs /target subdirectory.', + ) args = parser.parse_args() # Set up logging logging.basicConfig( format='%(message)s', level=int(args.loglevel) if args.loglevel.isdigit( ) else args.loglevel.upper(), ) # Read config generated by configure. config = configparser.ConfigParser() configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" config.read_file(open(configfile, encoding="utf8")) if not config["components"].getboolean("ENABLE_FUZZ"): logging.error("Must have fuzz targets built") sys.exit(1) test_dir = os.path.join( config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz') # Build list of tests test_list_all = [ f for f in os.listdir(test_dir) if os.path.isfile(os.path.join(test_dir, f)) and os.access(os.path.join(test_dir, f), os.X_OK)] if not test_list_all: logging.error("No fuzz targets found") sys.exit(1) logging.info("Fuzz targets found: {}".format(test_list_all)) # By default run all args.target = args.target or test_list_all test_list_error = list(set(args.target).difference(set(test_list_all))) if test_list_error: logging.error( "Unknown fuzz targets selected: {}".format(test_list_error)) test_list_selection = list( set(test_list_all).intersection(set(args.target))) if not test_list_selection: logging.error("No fuzz targets selected") logging.info("Fuzz targets selected: {}".format(test_list_selection)) test_list_seedless = [] for t in test_list_selection: corpus_path = os.path.join(args.seed_dir, t) if not os.path.exists(corpus_path) or len( os.listdir(corpus_path)) == 0: test_list_seedless.append(t) test_list_seedless.sort() if test_list_seedless: logging.info( "Fuzzing harnesses lacking a seed corpus: {}".format( " ".join(test_list_seedless) ) ) logging.info( "Please consider adding a fuzz seed corpus at https://github.com/Bitcoin-ABC/qa-assets") try: help_output = subprocess.run( args=[ os.path.join(test_dir, test_list_selection[0]), '-help=1', ], timeout=20, check=True, stderr=subprocess.PIPE, universal_newlines=True, ).stderr if "libFuzzer" not in help_output: logging.error("Must be built with libFuzzer") sys.exit(1) except subprocess.TimeoutExpired: logging.error( "subprocess timed out: Currently only libFuzzer is supported") sys.exit(1) + if args.m_dir: + merge_inputs( + corpus=args.seed_dir, + test_list=test_list_selection, + build_dir=config["environment"]["BUILDDIR"], + merge_dir=args.m_dir, + ) + run_once( corpus=args.seed_dir, test_list=test_list_selection, test_dir=test_dir, - export_coverage=args.export_coverage, use_valgrind=args.valgrind, ) -def run_once(*, corpus, test_list, test_dir, export_coverage, use_valgrind): +def merge_inputs(*, corpus, test_list, build_dir, merge_dir): + logging.info( + "Merge the inputs in the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) + for t in test_list: + args = [ + os.path.join(build_dir, 'src', 'test', 'fuzz', t), + '-merge=1', + os.path.join(corpus, t), + os.path.join(merge_dir, t), + ] + os.makedirs(os.path.join(corpus, t), exist_ok=True) + os.makedirs(os.path.join(merge_dir, t), exist_ok=True) + logging.debug('Run {} with args {}'.format(t, args)) + output = subprocess.run( + args, + check=True, + stderr=subprocess.PIPE, + universal_newlines=True).stderr + logging.debug('Output: {}'.format(output)) + + +def run_once(*, corpus, test_list, test_dir, use_valgrind): for t in test_list: corpus_path = os.path.join(corpus, t) os.makedirs(corpus_path, exist_ok=True) args = [ os.path.join(test_dir, t), '-runs=1', '-detect_leaks=0', corpus_path, ] if use_valgrind: args = [ 'valgrind', '--quiet', '--error-exitcode=1', '--exit-on-first-error=yes'] + args logging.debug('Run {} with args {}'.format(t, args)) result = subprocess.run( args, stderr=subprocess.PIPE, universal_newlines=True) output = result.stderr logging.debug('Output: {}'.format(output)) result.check_returncode() - if not export_coverage: - continue - for line in output.splitlines(): - if 'INITED' in line: - with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file: - cov_file.write(line) - break if __name__ == '__main__': main()