diff --git a/CMakeLists.txt b/CMakeLists.txt --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,6 +66,32 @@ "Dockerfile-doxygen" ) +option(ENABLE_COVERAGE "Enable coverage" OFF) +option(ENABLE_BRANCH_COVERAGE "Enable branch coverage" OFF) + +if(ENABLE_COVERAGE) + include(Coverage) + enable_coverage(${ENABLE_BRANCH_COVERAGE}) + set(CMAKE_BUILD_TYPE Coverage CACHE STRING + "Select the configuration for the build" FORCE) + + exclude_from_coverage( + "depends" + "src/bench" + "src/crypto/ctaes" + "src/leveldb" + "src/secp256k1" + "src/seeder" + "src/univalue" + ) + + add_custom_target_coverage(check) + add_custom_target_coverage(check-all) + add_custom_target_coverage(check-extended) + add_custom_target_coverage(check-upgrade-activated) + add_custom_target_coverage(check-upgrade-activated-extended) +endif() + add_subdirectory(src) add_subdirectory(test) diff --git a/cmake/modules/Coverage.cmake b/cmake/modules/Coverage.cmake new file mode 100644 --- /dev/null +++ b/cmake/modules/Coverage.cmake @@ -0,0 +1,149 @@ +# Copyright (c) 2020 The Bitcoin developers +############################################################################### +# Provide support for building test coverage reports. +# +# Dependencies: +# - c++filt +# - gcov +# - genhtml +# - lcov +# - python3 +# +# The basic workflow for building coverage report is as follow: +# 1/ Build the instrumented code by passing the --coverage flag to the +# compiler. For better accuracy there should be no optimization. +# 2/ Build the code, prior to run any test. Capture an initial state with +# `lcov` to get a baseline for the coverage information (function names, +# number of lines). This is achieved by running the `coverage-baseline` +# target. +# 3/ The coverage data is filtered to remove coverage info from code which is +# not of interest (such as 3rd party libs, generated files, etc.). +# 4/ Run the tests, and capture the generated coverage data with `lcov`. Again, +# the coverage data is filtered. After the test target has run, the +# coverage counters are reset in order to make each test run produce an +# independent output. +# 5/ Combine the test coverage data with the baseline data using `lcov` to +# produce the final coverage information for the test. +# 6/ Use `genhtml` to generate a HTML report from the combined coverage data. +# +# How to use: +# 1/ Enable coverage with the `enable_coverage()` function. The branch coverage +# can be enabled by setting the `ENABLE_BRANCH_COVERAGE` parameter to true. +# 2/ Call `exclude_from_coverage()` to exclude paths from the coverage report. +# 3/ Use `add_custom_target_coverage()` to enable coverage reporting for your +# test target. This generates a new `coverage-` target that will run +# the tests and create the HTML report in a `` directory. +############################################################################### + +include(SanitizeHelper) + +# Exclude directories (and subdirectories) from the coverage report. +function(exclude_from_coverage) + foreach(_dir ${ARGN}) + get_filename_component(_abspath "${_dir}" ABSOLUTE) + set_property(GLOBAL APPEND_STRING PROPERTY LCOV_FILTER_PATTERN " -p ${_abspath}") + endforeach() +endfunction() + +function(enable_coverage ENABLE_BRANCH_COVERAGE) + set(__ENABLE_COVERAGE ON CACHE INTERNAL "Coverage is enabled") + + # Required dependencies. + # c++filt is needed to demangle c++ function names at HTML generation time. + include(DoOrFail) + find_program_or_fail(LCOV_EXECUTABLE lcov) + find_program_or_fail(GCOV_EXECUTABLE gcov) + find_program_or_fail(GENHTML_EXECUTABLE genhtml) + find_program_or_fail(CXXFILT_EXECUTABLE c++filt) + find_package(Python3 COMPONENTS Interpreter REQUIRED) + set(__COVERAGE_PYTHON "${Python3_EXECUTABLE}" CACHE PATH "Path to the Python interpreter") + + get_property(_project_languages GLOBAL PROPERTY ENABLED_LANGUAGES) + set(COVERAGE_FLAG --coverage) + + foreach(_language ${_project_languages}) + sanitize_c_cxx_definition( + "supports_${_language}_" + ${COVERAGE_FLAG} + SUPPORTS_COVERAGE + ) + + set(_save_linker_flags ${CMAKE_EXE_LINKER_FLAGS}) + string(APPEND CMAKE_EXE_LINKER_FLAGS " ${COVERAGE_FLAG}") + + if("${_language}" STREQUAL "C") + include(CheckCCompilerFlag) + CHECK_C_COMPILER_FLAG(${COVERAGE_FLAG} ${SUPPORTS_COVERAGE}) + elseif("${_language}" STREQUAL "CXX") + include(CheckCXXCompilerFlag) + CHECK_CXX_COMPILER_FLAG(${COVERAGE_FLAG} ${SUPPORTS_COVERAGE}) + else() + message(WARNING "Coverage is not supported for the ${_language} language") + endif() + + set(CMAKE_EXE_LINKER_FLAGS ${_save_linker_flags}) + + if(NOT ${SUPPORTS_COVERAGE}) + message(FATAL_ERROR "The ${COVERAGE_FLAG} option is not supported by your ${_language} compiler") + endif() + + add_compile_options($<$:${COVERAGE_FLAG}>) + add_link_options($<$:${COVERAGE_FLAG}>) + endforeach() + + # Exclude some path by default, such as system headers and generated files. + exclude_from_coverage( + "${CMAKE_BINARY_DIR}" + "/usr/include" + "/usr/lib" + "/usr/lib64" + ) + + if(ENABLE_BRANCH_COVERAGE) + string(APPEND LCOV_OPTIONS "--rc lcov_branch_coverage=1") + set(LCOV_OPTIONS "${LCOV_OPTIONS}" CACHE STRING "Lcov options") + endif() +endfunction() + +function(add_custom_target_coverage TARGET) + # Coverage should have been enabled in order to create the new coverage-* + # targets. + if(NOT __ENABLE_COVERAGE) + return() + endif() + + get_property(LCOV_FILTER_PATTERN GLOBAL PROPERTY LCOV_FILTER_PATTERN) + + # Make sure we generate the base coverage data before building this target. + if(NOT TARGET coverage-baseline) + configure_file( + "${CMAKE_SOURCE_DIR}/cmake/templates/CoverageBaseline.sh.in" + "${CMAKE_BINARY_DIR}/CoverageBaseline.sh" + ) + + add_custom_command( + COMMENT "Generating baseline coverage info" + OUTPUT "${CMAKE_BINARY_DIR}/baseline.info" + COMMAND "${CMAKE_BINARY_DIR}/CoverageBaseline.sh" + VERBATIM USES_TERMINAL + ) + + add_custom_target(coverage-baseline + DEPENDS "${CMAKE_BINARY_DIR}/baseline.info" + ) + endif() + + sanitize_c_cxx_definition("" "${TARGET}" SANITIZED_TARGET) + + configure_file( + "${CMAKE_SOURCE_DIR}/cmake/templates/CoverageTest.sh.in" + "${CMAKE_BINARY_DIR}/Coverage-${TARGET}.sh" + ) + + add_custom_target(coverage-${TARGET} + DEPENDS coverage-baseline ${TARGET} + COMMENT "Generating ${TARGET} coverage report" + COMMAND "${CMAKE_BINARY_DIR}/Coverage-${TARGET}.sh" + VERBATIM USES_TERMINAL + ) +endfunction() diff --git a/cmake/modules/OverrideInitFlags.cmake b/cmake/modules/OverrideInitFlags.cmake --- a/cmake/modules/OverrideInitFlags.cmake +++ b/cmake/modules/OverrideInitFlags.cmake @@ -8,8 +8,19 @@ set(CMAKE_C_FLAGS_MINSIZEREL_INIT "-Os") set(CMAKE_C_FLAGS_RELEASE_INIT "-O3") set(CMAKE_C_FLAGS_RELWITHDEBINFO_INIT "-g -O2") +set(CMAKE_C_FLAGS_COVERAGE "-g -O0") set(CMAKE_CXX_FLAGS_DEBUG_INIT "-O0") set(CMAKE_CXX_FLAGS_MINSIZEREL_INIT "-Os") set(CMAKE_CXX_FLAGS_RELEASE_INIT "-O3") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO_INIT "-g -O2") +# Setting -Og instead of -O0 is a workaround for the GCC bug 90380: +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90380 +# +# This bug is fixed upstream, but is not widely distributed yet. +# Fixed in GCC versions: +# - GCC 7.x: versions <= 7.2 are unaffected +# - GCC 8.x: versions >= 8.3.1 +# - GCC 9.x: versions >= 9.1.1 +# - GCC 10.x: all versions +set(CMAKE_CXX_FLAGS_COVERAGE "-g -Og") diff --git a/cmake/modules/TestSuite.cmake b/cmake/modules/TestSuite.cmake --- a/cmake/modules/TestSuite.cmake +++ b/cmake/modules/TestSuite.cmake @@ -13,6 +13,8 @@ set(${TARGET} "check-${SUITE}") endmacro() +include(Coverage) + function(create_test_suite_with_parent_targets NAME) get_target_from_suite(${NAME} TARGET) @@ -26,6 +28,8 @@ add_dependencies(${PARENT_TARGET} ${TARGET}) endif() endforeach() + + add_custom_target_coverage(${TARGET}) endfunction() macro(create_test_suite NAME) diff --git a/cmake/templates/CoverageBaseline.sh.in b/cmake/templates/CoverageBaseline.sh.in new file mode 100755 --- /dev/null +++ b/cmake/templates/CoverageBaseline.sh.in @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +# Build the default target +"${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" + +# Capture (-c) initial (-i) coverage data in order to get a baseline +# before running any test. +"${LCOV_EXECUTABLE}" --gcov-tool="${GCOV_EXECUTABLE}" ${LCOV_OPTIONS} \ + -c -i -d "${CMAKE_BINARY_DIR}" \ + -o baseline_raw.info + +# Remove the coverage data for the paths matching any of the patterns. +"${__COVERAGE_PYTHON}" "${CMAKE_SOURCE_DIR}/cmake/utils/filter-lcov.py" \ + ${LCOV_FILTER_PATTERN} baseline_raw.info "${CMAKE_BINARY_DIR}/baseline.info" diff --git a/cmake/templates/CoverageTest.sh.in b/cmake/templates/CoverageTest.sh.in new file mode 100755 --- /dev/null +++ b/cmake/templates/CoverageTest.sh.in @@ -0,0 +1,30 @@ +#!/bin/sh + +set -e + +# Capture (-c) coverage data generated by the test. +"${LCOV_EXECUTABLE}" --gcov-tool="${GCOV_EXECUTABLE}" ${LCOV_OPTIONS} \ + -c -d "${CMAKE_BINARY_DIR}" \ + -t ${SANITIZED_TARGET} \ + -o "${TARGET}.info" + +# Reset to zero (-z) the counters (remove the *.gcda coverage files). +"${LCOV_EXECUTABLE}" --gcov-tool="${GCOV_EXECUTABLE}" ${LCOV_OPTIONS} \ + -z -d "${CMAKE_BINARY_DIR}" + +# Remove the coverage data for the paths matching any of the patterns. +"${__COVERAGE_PYTHON}" "${CMAKE_SOURCE_DIR}/cmake/utils/filter-lcov.py" \ + ${LCOV_FILTER_PATTERN} "${TARGET}.info" "${TARGET}_filtered.info" + +# Add (-a) the baseline and test coverage data files to combine them +# into a single one. +"${LCOV_EXECUTABLE}" --gcov-tool="${GCOV_EXECUTABLE}" ${LCOV_OPTIONS} \ + -a "${CMAKE_BINARY_DIR}/baseline.info" \ + -a "${TARGET}_filtered.info" \ + -o "${TARGET}_combined.info" + +# Generate the HTML coverage report from the coverage data. +"${GENHTML_EXECUTABLE}" ${LCOV_OPTIONS} \ + --demangle-cpp \ + -s "${TARGET}_combined.info" \ + -o "${CMAKE_BINARY_DIR}/${TARGET}.coverage" diff --git a/doc/developer-notes.md b/doc/developer-notes.md --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -283,20 +283,29 @@ ### Compiling for test coverage -LCOV can be used to generate a test coverage report based upon `make check` -execution. LCOV must be installed on your system (e.g. the `lcov` package -on Debian/Ubuntu). +LCOV can be used to generate a test coverage report based upon some test targets +execution. Some packages are required to generate the coverage report: +`c++filt`, `gcov`, `genhtml`, `lcov` and `python3`. -To enable LCOV report generation during test runs: +To install these dependencies on Debian 10: ```shell -./configure --enable-lcov -make -make cov +sudo apt install binutils-common g++ lcov python3 +``` + +To enable LCOV report generation during test runs: -# A coverage report will now be accessible at `./test_bitcoin.coverage/index.html`. +```shell +cmake -GNinja .. -DENABLE_COVERAGE=ON -DCCACHE=OFF +ninja coverage-check-all ``` +A coverage report will now be accessible at `./check-all.coverage/index.html`. + +To include branch coverage, you can add the `-DENABLE_BRANCH_COVERAGE=ON` option +to the `cmake` command line. + + ### Sanitizers Bitcoin ABC can be compiled with various "sanitizers" enabled, which add diff --git a/src/secp256k1/CMakeLists.txt b/src/secp256k1/CMakeLists.txt --- a/src/secp256k1/CMakeLists.txt +++ b/src/secp256k1/CMakeLists.txt @@ -6,13 +6,31 @@ # Add path for custom modules when building as a standalone project list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules) +option(SECP256K1_ENABLE_COVERAGE "Enable coverage" OFF) +option(SECP256K1_ENABLE_BRANCH_COVERAGE "Enable branch coverage" OFF) + +if(SECP256K1_ENABLE_COVERAGE) + include(Coverage) + + enable_coverage(${SECP256K1_ENABLE_BRANCH_COVERAGE}) + + file(GLOB COVERAGE_EXCLUDED_FILES src/bench*) + exclude_from_coverage(${COVERAGE_EXCLUDED_FILES}) + + set(CMAKE_BUILD_TYPE Coverage CACHE STRING + "Select the configuration for the build" FORCE) + + set(CMAKE_C_FLAGS_COVERAGE "-g -O0") + + set(COVERAGE 1) +endif() + # Default to RelWithDebInfo configuration if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Select the configuration for the build" FORCE) endif() -# TODO: use -O0 instead of -O3 when coverage is enabled set(CMAKE_C_FLAGS_RELWITHDEBINFO "-g -O3") include(AddCompilerFlags) @@ -335,12 +353,13 @@ endfunction() create_secp256k1_test(tests src/tests.c) - target_compile_definitions(tests PRIVATE VERIFY) - create_secp256k1_test(exhaustive_tests src/tests_exhaustive.c) + # This should not be enabled at the same time as coverage is. - # TODO: support coverage. - target_compile_definitions(exhaustive_tests PRIVATE VERIFY) + # The VERIFY failure branch is not expected to be reached, so it would make + # coverage appear lower if set. + target_compile_definitions(tests PRIVATE $<$>:VERIFY>) + target_compile_definitions(exhaustive_tests PRIVATE $<$>:VERIFY>) if(SECP256K1_ENABLE_JNI) set(SECP256k1_JNI_TEST_JAR "secp256k1-jni-test") diff --git a/src/secp256k1/README.md b/src/secp256k1/README.md --- a/src/secp256k1/README.md +++ b/src/secp256k1/README.md @@ -101,7 +101,9 @@ This library aims to have full coverage of the reachable lines and branches. -To create a test coverage report, configure with `--enable-coverage` (use of GCC is necessary): +__To create a test coverage report with autotools:__ + +Configure with `--enable-coverage` (use of GCC is necessary): $ ./configure --enable-coverage @@ -117,6 +119,27 @@ $ gcovr --exclude 'src/bench*' --html --html-details -o coverage.html + +__To create a test coverage report with CMake:__ + +Make sure you installed the dependencies first, and they are in your `PATH`: +`c++filt`, `gcov`, `genhtml`, `lcov` and `python3`. + +Then run the build, tests and generate the coverage report with: + +```bash +mkdir coverage +cd coverage +cmake -GNinja .. \ + -DCMAKE_C_COMPILER=gcc \ + -DSECP256K1_ENABLE_BRANCH_COVERAGE=ON \ + -DSECP256K1_ENABLE_BRANCH_COVERAGE=ON # optional +ninja coverage-check-secp256k1 +``` + +The coverage report will be available by opening the file +`check-secp256k1.coverage/index.html` with a web browser. + Reporting a vulnerability ------------ diff --git a/src/secp256k1/src/libsecp256k1-config.h.cmake.in b/src/secp256k1/src/libsecp256k1-config.h.cmake.in --- a/src/secp256k1/src/libsecp256k1-config.h.cmake.in +++ b/src/secp256k1/src/libsecp256k1-config.h.cmake.in @@ -36,4 +36,6 @@ #cmakedefine ENABLE_OPENSSL_TESTS +#cmakedefine COVERAGE + #endif /* LIBSECP256K1_CONFIG_H */ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -46,6 +46,7 @@ make_link(util/bitcoin-util-test.py) make_link(util/rpcauth-test.py) +include(Coverage) macro(add_functional_test_check TARGET COMMENT) add_custom_target(${TARGET} COMMENT "${COMMENT}" @@ -60,6 +61,8 @@ USES_TERMINAL VERBATIM ) + + add_custom_target_coverage(${TARGET}) endmacro() add_functional_test_check(check-functional