diff --git a/CMakeLists.txt b/CMakeLists.txt --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Select the configuration for the build" FORCE) + set(__NO_USER_CMAKE_BUILD_TYPE ON CACHE BOOL "True if the user didn't set a build type on the command line") endif() # Find the python interpreter. This is required for several targets. @@ -66,6 +67,49 @@ "Dockerfile-doxygen" ) +option(ENABLE_COVERAGE "Enable coverage" OFF) +option(ENABLE_BRANCH_COVERAGE "Enable branch coverage" OFF) + +if(ENABLE_COVERAGE) + include(Coverage) + enable_coverage(${ENABLE_BRANCH_COVERAGE}) + + include(AddCompilerFlags) + + # If no build type is manually defined, override the optimization level. + # Otherwise, alert the user than the coverage result might be useless. + if(__NO_USER_CMAKE_BUILD_TYPE) + set_c_optimization_level(0) + + # Setting -Og instead of -O0 is a workaround for the GCC bug 90380: + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90380 + # + # This bug is fixed upstream, but is not widely distributed yet. + # Fixed in GCC versions: + # - GCC 7.x: versions <= 7.2 are unaffected + # - GCC 8.x: versions >= 8.3.1 + # - GCC 9.x: versions >= 9.1.1 + # - GCC 10.x: all versions + set_cxx_optimization_level(g) + else() + message(WARNING "It is advised to not enforce CMAKE_BUILD_TYPE to get the best coverage results") + endif() + + exclude_from_coverage( + "depends" + "src/bench" + "src/crypto/ctaes" + "src/leveldb" + "src/univalue" + ) + + add_custom_target_coverage(check) + add_custom_target_coverage(check-all) + add_custom_target_coverage(check-extended) + add_custom_target_coverage(check-upgrade-activated) + add_custom_target_coverage(check-upgrade-activated-extended) +endif() + add_subdirectory(src) add_subdirectory(test) diff --git a/cmake/modules/AddCompilerFlags.cmake b/cmake/modules/AddCompilerFlags.cmake --- a/cmake/modules/AddCompilerFlags.cmake +++ b/cmake/modules/AddCompilerFlags.cmake @@ -141,3 +141,26 @@ endif() endforeach() endfunction() + +macro(remove_optimization_level_from_var VAR) + string(REGEX REPLACE "-O[0-3gs]( |$)" "" ${VAR} "${${VAR}}") +endmacro() + +function(set_optimization_level_for_language LANGUAGE LEVEL) + if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "") + string(TOUPPER "CMAKE_${LANGUAGE}_FLAGS_${CMAKE_BUILD_TYPE}" BUILD_TYPE_FLAGS) + remove_optimization_level_from_var(${BUILD_TYPE_FLAGS}) + set(${BUILD_TYPE_FLAGS} "${${BUILD_TYPE_FLAGS}}" PARENT_SCOPE) + endif() + + remove_optimization_level_from_var(CMAKE_${LANGUAGE}_FLAGS) + set(CMAKE_${LANGUAGE}_FLAGS "${CMAKE_${LANGUAGE}_FLAGS} -O${LEVEL}" PARENT_SCOPE) +endfunction() + +macro(set_c_optimization_level LEVEL) + set_optimization_level_for_language(C ${LEVEL}) +endmacro() + +macro(set_cxx_optimization_level LEVEL) + set_optimization_level_for_language(CXX ${LEVEL}) +endmacro() diff --git a/cmake/modules/Coverage.cmake b/cmake/modules/Coverage.cmake new file mode 100644 --- /dev/null +++ b/cmake/modules/Coverage.cmake @@ -0,0 +1,149 @@ +# Copyright (c) 2020 The Bitcoin developers +############################################################################### +# Provide support for building test coverage reports. +# +# Dependencies: +# - c++filt +# - gcov +# - genhtml +# - lcov +# - python3 +# +# The basic workflow for building coverage report is as follow: +# 1/ Build the instrumented code by passing the --coverage flag to the +# compiler. For better accuracy there should be no optimization. +# 2/ Build the code, prior to run any test. Capture an initial state with +# `lcov` to get a baseline for the coverage information (function names, +# number of lines). This is achieved by running the `coverage-baseline` +# target. +# 3/ The coverage data is filtered to remove coverage info from code which is +# not of interest (such as 3rd party libs, generated files, etc.). +# 4/ Run the tests, and capture the generated coverage data with `lcov`. Again, +# the coverage data is filtered. After the test target has run, the +# coverage counters are reset in order to make each test run produce an +# independent output. +# 5/ Combine the test coverage data with the baseline data using `lcov` to +# produce the final coverage information for the test. +# 6/ Use `genhtml` to generate a HTML report from the combined coverage data. +# +# How to use: +# 1/ Enable coverage with the `enable_coverage()` function. The branch coverage +# can be enabled by setting the `ENABLE_BRANCH_COVERAGE` parameter to true. +# 2/ Call `exclude_from_coverage()` to exclude paths from the coverage report. +# 3/ Use `add_custom_target_coverage()` to enable coverage reporting for your +# test target. This generates a new `coverage-` target that will run +# the tests and create the HTML report in a `` directory. +############################################################################### + +include(SanitizeHelper) + +# Exclude directories (and subdirectories) from the coverage report. +function(exclude_from_coverage) + foreach(_dir ${ARGN}) + get_filename_component(_abspath "${_dir}" ABSOLUTE) + set_property(GLOBAL APPEND_STRING PROPERTY LCOV_FILTER_PATTERN " -p ${_abspath}") + endforeach() +endfunction() + +function(enable_coverage ENABLE_BRANCH_COVERAGE) + set(__ENABLE_COVERAGE ON CACHE INTERNAL "Coverage is enabled") + + # Required dependencies. + # c++filt is needed to demangle c++ function names at HTML generation time. + include(DoOrFail) + find_program_or_fail(LCOV_EXECUTABLE lcov) + find_program_or_fail(GCOV_EXECUTABLE gcov) + find_program_or_fail(GENHTML_EXECUTABLE genhtml) + find_program_or_fail(CXXFILT_EXECUTABLE c++filt) + find_package(Python3 COMPONENTS Interpreter REQUIRED) + set(__COVERAGE_PYTHON "${Python3_EXECUTABLE}" CACHE PATH "Path to the Python interpreter") + + get_property(_project_languages GLOBAL PROPERTY ENABLED_LANGUAGES) + set(COVERAGE_FLAG --coverage) + + foreach(_language ${_project_languages}) + sanitize_c_cxx_definition( + "supports_${_language}_" + ${COVERAGE_FLAG} + SUPPORTS_COVERAGE + ) + + set(_save_linker_flags ${CMAKE_EXE_LINKER_FLAGS}) + string(APPEND CMAKE_EXE_LINKER_FLAGS " ${COVERAGE_FLAG}") + + if("${_language}" STREQUAL "C") + include(CheckCCompilerFlag) + CHECK_C_COMPILER_FLAG(${COVERAGE_FLAG} ${SUPPORTS_COVERAGE}) + elseif("${_language}" STREQUAL "CXX") + include(CheckCXXCompilerFlag) + CHECK_CXX_COMPILER_FLAG(${COVERAGE_FLAG} ${SUPPORTS_COVERAGE}) + else() + message(WARNING "Coverage is not supported for the ${_language} language") + endif() + + set(CMAKE_EXE_LINKER_FLAGS ${_save_linker_flags}) + + if(NOT ${SUPPORTS_COVERAGE}) + message(FATAL_ERROR "The ${COVERAGE_FLAG} option is not supported by your ${_language} compiler") + endif() + + add_compile_options($<$:${COVERAGE_FLAG}>) + add_link_options($<$:${COVERAGE_FLAG}>) + endforeach() + + # Exclude some path by default, such as system headers and generated files. + exclude_from_coverage( + "${CMAKE_BINARY_DIR}" + "/usr/include" + "/usr/lib" + "/usr/lib64" + ) + + if(ENABLE_BRANCH_COVERAGE) + string(APPEND LCOV_OPTIONS "--rc lcov_branch_coverage=1") + set(LCOV_OPTIONS "${LCOV_OPTIONS}" CACHE STRING "Lcov options") + endif() +endfunction() + +function(add_custom_target_coverage TARGET) + # Coverage should have been enabled in order to create the new coverage-* + # targets. + if(NOT __ENABLE_COVERAGE) + return() + endif() + + get_property(LCOV_FILTER_PATTERN GLOBAL PROPERTY LCOV_FILTER_PATTERN) + + # Make sure we generate the base coverage data before building this target. + if(NOT TARGET coverage-baseline) + configure_file( + "${CMAKE_SOURCE_DIR}/cmake/templates/CoverageBaseline.sh.in" + "${CMAKE_BINARY_DIR}/CoverageBaseline.sh" + ) + + add_custom_command( + COMMENT "Generating baseline coverage info" + OUTPUT "${CMAKE_BINARY_DIR}/baseline.info" + COMMAND "${CMAKE_BINARY_DIR}/CoverageBaseline.sh" + VERBATIM USES_TERMINAL + ) + + add_custom_target(coverage-baseline + DEPENDS "${CMAKE_BINARY_DIR}/baseline.info" + ) + endif() + + sanitize_c_cxx_definition("" "${TARGET}" SANITIZED_TARGET) + + configure_file( + "${CMAKE_SOURCE_DIR}/cmake/templates/CoverageTest.sh.in" + "${CMAKE_BINARY_DIR}/Coverage-${TARGET}.sh" + ) + + add_custom_target(coverage-${TARGET} + DEPENDS coverage-baseline ${TARGET} + COMMENT "Generating ${TARGET} coverage report" + COMMAND "${CMAKE_BINARY_DIR}/Coverage-${TARGET}.sh" + VERBATIM USES_TERMINAL + ) +endfunction() diff --git a/cmake/modules/TestSuite.cmake b/cmake/modules/TestSuite.cmake --- a/cmake/modules/TestSuite.cmake +++ b/cmake/modules/TestSuite.cmake @@ -28,6 +28,8 @@ set(${TARGET} "check-${SUITE}") endmacro() +include(Coverage) + function(create_test_suite_with_parent_targets NAME) get_target_from_suite(${NAME} TARGET) @@ -41,6 +43,8 @@ add_dependencies(${PARENT_TARGET} ${TARGET}) endif() endforeach() + + add_custom_target_coverage(${TARGET}) endfunction() macro(create_test_suite NAME) diff --git a/cmake/templates/CoverageBaseline.sh.in b/cmake/templates/CoverageBaseline.sh.in new file mode 100755 --- /dev/null +++ b/cmake/templates/CoverageBaseline.sh.in @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +# Build the default target +"${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" + +# Capture (-c) initial (-i) coverage data in order to get a baseline +# before running any test. +"${LCOV_EXECUTABLE}" --gcov-tool="${GCOV_EXECUTABLE}" ${LCOV_OPTIONS} \ + -c -i -d "${CMAKE_BINARY_DIR}" \ + -o baseline_raw.info + +# Remove the coverage data for the paths matching any of the patterns. +"${__COVERAGE_PYTHON}" "${CMAKE_SOURCE_DIR}/cmake/utils/filter-lcov.py" \ + ${LCOV_FILTER_PATTERN} baseline_raw.info "${CMAKE_BINARY_DIR}/baseline.info" diff --git a/cmake/templates/CoverageTest.sh.in b/cmake/templates/CoverageTest.sh.in new file mode 100755 --- /dev/null +++ b/cmake/templates/CoverageTest.sh.in @@ -0,0 +1,30 @@ +#!/bin/sh + +set -e + +# Capture (-c) coverage data generated by the test. +"${LCOV_EXECUTABLE}" --gcov-tool="${GCOV_EXECUTABLE}" ${LCOV_OPTIONS} \ + -c -d "${CMAKE_BINARY_DIR}" \ + -t ${SANITIZED_TARGET} \ + -o "${TARGET}.info" + +# Reset to zero (-z) the counters (remove the *.gcda coverage files). +"${LCOV_EXECUTABLE}" --gcov-tool="${GCOV_EXECUTABLE}" ${LCOV_OPTIONS} \ + -z -d "${CMAKE_BINARY_DIR}" + +# Remove the coverage data for the paths matching any of the patterns. +"${__COVERAGE_PYTHON}" "${CMAKE_SOURCE_DIR}/cmake/utils/filter-lcov.py" \ + ${LCOV_FILTER_PATTERN} "${TARGET}.info" "${TARGET}_filtered.info" + +# Add (-a) the baseline and test coverage data files to combine them +# into a single one. +"${LCOV_EXECUTABLE}" --gcov-tool="${GCOV_EXECUTABLE}" ${LCOV_OPTIONS} \ + -a "${CMAKE_BINARY_DIR}/baseline.info" \ + -a "${TARGET}_filtered.info" \ + -o "${TARGET}_combined.info" + +# Generate the HTML coverage report from the coverage data. +"${GENHTML_EXECUTABLE}" ${LCOV_OPTIONS} \ + --demangle-cpp \ + -s "${TARGET}_combined.info" \ + -o "${CMAKE_BINARY_DIR}/${TARGET}.coverage" diff --git a/doc/developer-notes.md b/doc/developer-notes.md --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -283,20 +283,29 @@ ### Compiling for test coverage -LCOV can be used to generate a test coverage report based upon `make check` -execution. LCOV must be installed on your system (e.g. the `lcov` package -on Debian/Ubuntu). +LCOV can be used to generate a test coverage report based upon some test targets +execution. Some packages are required to generate the coverage report: +`c++filt`, `gcov`, `genhtml`, `lcov` and `python3`. -To enable LCOV report generation during test runs: +To install these dependencies on Debian 10: ```shell -./configure --enable-lcov -make -make cov +sudo apt install binutils-common g++ lcov python3 +``` + +To enable LCOV report generation during test runs: -# A coverage report will now be accessible at `./test_bitcoin.coverage/index.html`. +```shell +cmake -GNinja .. -DENABLE_COVERAGE=ON +ninja coverage-check-all ``` +A coverage report will now be accessible at `./check-all.coverage/index.html`. + +To include branch coverage, you can add the `-DENABLE_BRANCH_COVERAGE=ON` option +to the `cmake` command line. + + ### Sanitizers Bitcoin ABC can be compiled with various "sanitizers" enabled, which add diff --git a/src/secp256k1/CMakeLists.txt b/src/secp256k1/CMakeLists.txt --- a/src/secp256k1/CMakeLists.txt +++ b/src/secp256k1/CMakeLists.txt @@ -10,13 +10,36 @@ if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Select the configuration for the build" FORCE) + set(__NO_USER_CMAKE_BUILD_TYPE ON CACHE BOOL "True if the user didn't set a build type on the command line") endif() -# TODO: use -O0 instead of -O3 when coverage is enabled set(CMAKE_C_FLAGS_RELWITHDEBINFO "-g -O3") +option(SECP256K1_ENABLE_COVERAGE "Enable coverage" OFF) +option(SECP256K1_ENABLE_BRANCH_COVERAGE "Enable branch coverage" OFF) + include(AddCompilerFlags) +if(SECP256K1_ENABLE_COVERAGE) + include(Coverage) + + enable_coverage(${SECP256K1_ENABLE_BRANCH_COVERAGE}) + + exclude_from_coverage("${CMAKE_CURRENT_SOURCE_DIR}/src/bench") + + # If no build type is manually defined, override the optimization level. + # Otherwise, alert the user than the coverage result might be useless. + if(__NO_USER_CMAKE_BUILD_TYPE) + set_c_optimization_level(0) + else() + message(WARNING "It is advised to not enforce CMAKE_BUILD_TYPE to get the best coverage results") + endif() + + set(COVERAGE 1) +endif() + + + # libsecp256k1 use a different set of flags. add_c_compiler_flags( -pedantic @@ -346,12 +369,15 @@ endfunction() create_secp256k1_test(tests src/tests.c) - target_compile_definitions(tests PRIVATE VERIFY) - create_secp256k1_test(exhaustive_tests src/tests_exhaustive.c) + # This should not be enabled at the same time as coverage is. - # TODO: support coverage. - target_compile_definitions(exhaustive_tests PRIVATE VERIFY) + # The VERIFY failure branch is not expected to be reached, so it would make + # coverage appear lower if set. + if(NOT SECP256K1_ENABLE_COVERAGE) + target_compile_definitions(tests PRIVATE VERIFY) + target_compile_definitions(exhaustive_tests PRIVATE VERIFY) + endif() if(SECP256K1_ENABLE_JNI) set(SECP256k1_JNI_TEST_JAR "secp256k1-jni-test") diff --git a/src/secp256k1/README.md b/src/secp256k1/README.md --- a/src/secp256k1/README.md +++ b/src/secp256k1/README.md @@ -101,7 +101,9 @@ This library aims to have full coverage of the reachable lines and branches. -To create a test coverage report, configure with `--enable-coverage` (use of GCC is necessary): +__To create a test coverage report with autotools:__ + +Configure with `--enable-coverage` (use of GCC is necessary): $ ./configure --enable-coverage @@ -117,6 +119,27 @@ $ gcovr --exclude 'src/bench*' --html --html-details -o coverage.html + +__To create a test coverage report with CMake:__ + +Make sure you installed the dependencies first, and they are in your `PATH`: +`c++filt`, `gcov`, `genhtml`, `lcov` and `python3`. + +Then run the build, tests and generate the coverage report with: + +```bash +mkdir coverage +cd coverage +cmake -GNinja .. \ + -DCMAKE_C_COMPILER=gcc \ + -DSECP256K1_ENABLE_COVERAGE=ON \ + -DSECP256K1_ENABLE_BRANCH_COVERAGE=ON # optional +ninja coverage-check-secp256k1 +``` + +The coverage report will be available by opening the file +`check-secp256k1.coverage/index.html` with a web browser. + Reporting a vulnerability ------------ diff --git a/src/secp256k1/src/libsecp256k1-config.h.cmake.in b/src/secp256k1/src/libsecp256k1-config.h.cmake.in --- a/src/secp256k1/src/libsecp256k1-config.h.cmake.in +++ b/src/secp256k1/src/libsecp256k1-config.h.cmake.in @@ -36,4 +36,6 @@ #cmakedefine ENABLE_OPENSSL_TESTS +#cmakedefine COVERAGE + #endif /* LIBSECP256K1_CONFIG_H */ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -52,6 +52,7 @@ make_link(util/rpcauth-test.py) make_link(fuzz/test_runner.py) +include(Coverage) include(TestSuite) macro(add_functional_test_check TARGET COMMENT) add_test_custom_target(${TARGET} @@ -68,6 +69,8 @@ USES_TERMINAL VERBATIM ) + + add_custom_target_coverage(${TARGET}) endmacro() add_functional_test_check(check-functional