diff --git a/CMakeLists.txt b/CMakeLists.txt --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,6 +68,32 @@ "Dockerfile-doxygen" ) +option(ENABLE_BRANCH_COVERAGE "Enable branch coverage" OFF) + +if(CMAKE_BUILD_TYPE STREQUAL "Coverage") + include(Coverage) + + if(ENABLE_BRANCH_COVERAGE) + enable_branch_coverage() + endif() + + exclude_from_coverage( + "depends" + "src/bench" + "src/crypto/ctaes" + "src/leveldb" + "src/secp256k1" + "src/seeder" + "src/univalue" + ) + + add_custom_target_coverage(check) + add_custom_target_coverage(check-all) + add_custom_target_coverage(check-extended) + add_custom_target_coverage(check-upgrade-activated) + add_custom_target_coverage(check-upgrade-activated-extended) +endif() + add_subdirectory(src) add_subdirectory(test) diff --git a/cmake/modules/Coverage.cmake b/cmake/modules/Coverage.cmake new file mode 100644 --- /dev/null +++ b/cmake/modules/Coverage.cmake @@ -0,0 +1,273 @@ +# Copyright (c) 2020 The Bitcoin developers +############################################################################### +# Provide support for building test coverage reports. +# +# Dependencies: +# - c++filt +# - gcov +# - genhtml +# - lcov +# - python3 +# +# The basic workflow for building coverage report is as follow: +# 1/ Build the instrumented code by passing the --coverage flag to the +# compiler. For better accuracy there should be no optimization. +# 2/ Build the code, prior to run any test. Capture an initial state with +# `lcov` to get a baseline for the coverage information (function names, +# number of lines). This is achieved by running the `coverage-baseline` +# target (see `_generate_coverage_baseline()`). +# 3/ The coverage data is filtered to remove coverage info from code which is +# not of interest (such as 3rd party libs, generated files, etc.). +# 4/ Run the tests, and capture the generated coverage data with `lcov`. Again, +# the coverage data is filtered. After the test target has run, the +# coverage counters are reset in order to make each test run produce an +# independent output (see `_add_target_coverage()`). +# 5/ Combine the test coverage data with the baseline data using `lcov` to +# produce the final coverage information for the test (see +# `_combine_test_info_with_baseline()`). +# 6/ Use `genhtml` to generate a HTML report from the combined coverage data. +# The coverage info is deleted once the report has been generated, so that +# subsequent run will regenerate this data (see `_generate_html_report()`). +# +# How to use: +# 1/ Include this file. This will ensure that the environment is compatible +# with coverage report generation. +# 2/ Setup your project otions: +# - Call `enable_branch_coverage()` to report branch coverage information. +# - Call `exclude_from_coverage()` to exclude a path from the coverage +# report. +# 3/ Use `add_custom_target_coverage()` and `add_executable_coverage()` to +# enable coverage reporting for your test target. This generates a new +# `coverage-` target that will run the tests and create the HTML +# report in a `` directory. +############################################################################### + + +### API + +macro(enable_branch_coverage) + set(LCOV_OPTIONS --rc lcov_branch_coverage=1) +endmacro() + +function(exclude_from_coverage) + foreach(_dir ${ARGN}) + get_filename_component(_abspath "${_dir}" ABSOLUTE) + set_property(GLOBAL APPEND PROPERTY COVERAGE_EXCLUDED_DIRS "${_abspath}") + endforeach() +endfunction() + +macro(add_custom_target_coverage TARGET) + _add_target_coverage(${TARGET} + COMMAND + "${CMAKE_COMMAND}" + --build "${CMAKE_BINARY_DIR}" + --target ${TARGET} + ) +endmacro() + +macro(add_executable_coverage TARGET) + # The TARGET executable path is expanded and the dependency added + # automatically by cmake. + _add_target_coverage(${TARGET} COMMAND ${TARGET}) + + # Make the target sources part of the baseline data. + add_dependencies(${COVERAGE_BASELINE_TARGET} ${TARGET}) +endmacro() + + +### INTERNALS + +# Required dependencies. +# c++filt is needed to demangle c++ function names at HTML generation time. +include(DoOrFail) +find_program_or_fail(LCOV_EXECUTABLE lcov) +find_program_or_fail(GCOV_EXECUTABLE gcov) +find_program_or_fail(GENHTML_EXECUTABLE genhtml) +find_program_or_fail(CXXFILT_EXECUTABLE c++filt) +find_package(Python 3.5 COMPONENTS Interpreter REQUIRED) + +# Set the --coverage flag if supported by the user environment, abort otherwise. +function(_set_coverage_flag_if_supported COVERAGE_FLAG) + get_property(_project_languages GLOBAL PROPERTY ENABLED_LANGUAGES) + + foreach(_language ${_project_languages}) + include(SanitizeHelper) + sanitize_c_cxx_definition( + "supports_${_language}_" + ${COVERAGE_FLAG} + SUPPORTS_COVERAGE + ) + + set(_save_linker_flags ${CMAKE_EXE_LINKER_FLAGS}) + string(APPEND CMAKE_EXE_LINKER_FLAGS " ${COVERAGE_FLAG}") + + if("${_language}" STREQUAL "C") + include(CheckCCompilerFlag) + CHECK_C_COMPILER_FLAG(${COVERAGE_FLAG} ${SUPPORTS_COVERAGE}) + elseif("${_language}" STREQUAL "CXX") + include(CheckCXXCompilerFlag) + CHECK_CXX_COMPILER_FLAG(${COVERAGE_FLAG} ${SUPPORTS_COVERAGE}) + else() + message(FATAL_ERROR + "Coverage is not supported for the ${_language} language") + endif() + + set(CMAKE_EXE_LINKER_FLAGS ${_save_linker_flags}) + + if(NOT ${SUPPORTS_COVERAGE}) + message(FATAL_ERROR + "The ${COVERAGE_FLAG} option is not supported by your ${_language} compiler" + ) + endif() + + add_compile_options($<$:${COVERAGE_FLAG}>) + add_link_options($<$:${COVERAGE_FLAG}>) + endforeach() +endfunction() + +_set_coverage_flag_if_supported(--coverage) + +# Exclude some path by default, such as system headers and generated files. +exclude_from_coverage( + "${CMAKE_BINARY_DIR}" + "/usr/include" + "/usr/lib" + "/usr/lib64" +) + +set(LCOV_COMMAND "${LCOV_EXECUTABLE}" "--gcov-tool=${GCOV_EXECUTABLE}") +set(COVERAGE_BASELINE_INFO_FILE "${CMAKE_BINARY_DIR}/baseline.info") +set(COVERAGE_BASELINE_TARGET "coverage-baseline") + +function(_filter_coverage_info RAW_FILE FILTERED_FILE) + get_property(EXCLUDED_DIRS GLOBAL PROPERTY COVERAGE_EXCLUDED_DIRS) + list(REMOVE_DUPLICATES EXCLUDED_DIRS) + foreach(_dir ${EXCLUDED_DIRS}) + if(NOT "${_dir}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") + list(APPEND LCOV_FILTER_PATTERN -p "${_dir}") + endif() + endforeach() + + add_custom_command( + COMMENT "Filtering ${RAW_FILE} coverage data" + OUTPUT "${FILTERED_FILE}" + DEPENDS "${RAW_FILE}" + # Remove the coverage data for the paths matching any of the patterns + COMMAND + "${Python_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/cmake/utils/filter-lcov.py" + ${LCOV_FILTER_PATTERN} "${RAW_FILE}" "${FILTERED_FILE}" + COMMAND + "${CMAKE_COMMAND}" -E remove "${RAW_FILE}" + VERBATIM USES_TERMINAL + ) +endfunction() + +function(_generate_coverage_baseline) + set(BASELINE_RAW_INFO_FILE "${CMAKE_BINARY_DIR}/baseline_raw.info") + + add_custom_command( + COMMENT "Generating baseline coverage info" + OUTPUT "${BASELINE_RAW_INFO_FILE}" + # Build the default target + COMMAND + "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" + # Capture (-c) initial (-i) coverage data in order to get a baseline + # before running any test. + COMMAND + ${LCOV_COMMAND} ${LCOV_OPTIONS} + -c -i -d "${CMAKE_BINARY_DIR}" + -o "${BASELINE_RAW_INFO_FILE}" + VERBATIM USES_TERMINAL + ) + + _filter_coverage_info(${BASELINE_RAW_INFO_FILE} ${COVERAGE_BASELINE_INFO_FILE}) + + add_custom_target(${COVERAGE_BASELINE_TARGET} + DEPENDS "${COVERAGE_BASELINE_INFO_FILE}" + ) +endfunction() + +macro(_combine_test_info_with_baseline TEST_INFO COMBINED_INFO) + add_custom_command( + COMMENT "Combining ${TEST_INFO} coverage data with baseline coverage data" + OUTPUT "${COMBINED_INFO}" + DEPENDS "${COVERAGE_BASELINE_INFO_FILE}" "${TEST_INFO}" + # Add (-a) several coverage data files to combine them into a single one + COMMAND + ${LCOV_COMMAND} ${LCOV_OPTIONS} + -a "${COVERAGE_BASELINE_INFO_FILE}" + -a "${TEST_INFO}" + -o "${COMBINED_INFO}" + COMMAND + "${CMAKE_COMMAND}" -E remove "${TEST_INFO}" + VERBATIM USES_TERMINAL + ) +endmacro() + +macro(_generate_html_report COVERAGE_INFO COVERAGE_REPORT_DIR) + add_custom_command( + COMMENT "Generating HTML coverage report from ${COVERAGE_INFO}" + OUTPUT "${COVERAGE_REPORT_DIR}" + DEPENDS "${COVERAGE_INFO}" + # Generate the HTML coverage report from the coverage data + COMMAND + "${GENHTML_EXECUTABLE}" ${LCOV_OPTIONS} + --demangle-cpp + -s "${COVERAGE_INFO}" + -o "${COVERAGE_REPORT_DIR}" + COMMAND + "${CMAKE_COMMAND}" -E remove "${COVERAGE_INFO}" + VERBATIM USES_TERMINAL + ) +endmacro() + +function(_add_target_coverage TARGET) + sanitize_c_cxx_definition("" "${TARGET}" SANITIZED_TARGET) + set(INFO_FILE + "${CMAKE_CURRENT_BINARY_DIR}/${SANITIZED_TARGET}_raw.info" + ) + set(FILTERED_INFO_FILE + "${CMAKE_CURRENT_BINARY_DIR}/${SANITIZED_TARGET}.info" + ) + set(COMBINED_INFO_FILE + "${CMAKE_CURRENT_BINARY_DIR}/${SANITIZED_TARGET}_coverage.info" + ) + set(COVERAGE_DIR + "${CMAKE_BINARY_DIR}/${SANITIZED_TARGET}.coverage" + ) + + if(NOT TARGET ${COVERAGE_BASELINE_TARGET}) + _generate_coverage_baseline() + endif() + + add_custom_command( + COMMENT "Generating ${INFO_FILE} coverage info" + OUTPUT "${INFO_FILE}" + # Make sure the baseline info is generated before running any test + DEPENDS ${COVERAGE_BASELINE_TARGET} + # Command to run the test + ${ARGN} + # Capture (-c) coverage data generated by the test + COMMAND + ${LCOV_COMMAND} ${LCOV_OPTIONS} + -c -d "${CMAKE_BINARY_DIR}" + -t "${SANITIZED_TARGET}" + -o "${INFO_FILE}" + # Reset to zero (-z) the counters (remove the *.gcda coverage files) + COMMAND + ${LCOV_COMMAND} ${LCOV_OPTIONS} + -z -d "${CMAKE_BINARY_DIR}" + VERBATIM USES_TERMINAL + ) + + _filter_coverage_info("${INFO_FILE}" "${FILTERED_INFO_FILE}") + + _combine_test_info_with_baseline( + "${FILTERED_INFO_FILE}" + "${COMBINED_INFO_FILE}" + ) + + _generate_html_report("${COMBINED_INFO_FILE}" "${COVERAGE_DIR}") + + add_custom_target("coverage-${TARGET}" DEPENDS "${COVERAGE_DIR}") +endfunction() diff --git a/cmake/modules/OverrideInitFlags.cmake b/cmake/modules/OverrideInitFlags.cmake --- a/cmake/modules/OverrideInitFlags.cmake +++ b/cmake/modules/OverrideInitFlags.cmake @@ -8,8 +8,19 @@ set(CMAKE_C_FLAGS_MINSIZEREL_INIT "-Os") set(CMAKE_C_FLAGS_RELEASE_INIT "-O3") set(CMAKE_C_FLAGS_RELWITHDEBINFO_INIT "-g -O2") +set(CMAKE_C_FLAGS_COVERAGE "-g -O0") set(CMAKE_CXX_FLAGS_DEBUG_INIT "-O0") set(CMAKE_CXX_FLAGS_MINSIZEREL_INIT "-Os") set(CMAKE_CXX_FLAGS_RELEASE_INIT "-O3") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO_INIT "-g -O2") +# Setting -Og instead of -O0 is a workaround for the GCC bug 90380: +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90380 +# +# This bug is fixed upstream, but is not widely distributed yet. +# Fixed in GCC versions: +# - GCC 7.x: versions <= 7.2 are unaffected +# - GCC 8.x: versions >= 8.3.1 +# - GCC 9.x: versions >= 9.1.1 +# - GCC 10.x: all versions +set(CMAKE_CXX_FLAGS_COVERAGE "-g -Og") diff --git a/cmake/modules/TestSuite.cmake b/cmake/modules/TestSuite.cmake --- a/cmake/modules/TestSuite.cmake +++ b/cmake/modules/TestSuite.cmake @@ -26,6 +26,11 @@ add_dependencies(${PARENT_TARGET} ${TARGET}) endif() endforeach() + + if(CMAKE_BUILD_TYPE STREQUAL "Coverage") + include(Coverage) + add_custom_target_coverage(${TARGET}) + endif() endfunction() macro(create_test_suite NAME) @@ -56,8 +61,16 @@ add_dependencies(${SUITE_TARGET} ${TARGET}) endfunction() -function(add_test_to_suite SUITE NAME) +macro(add_test_executable NAME) add_executable(${NAME} EXCLUDE_FROM_ALL ${ARGN}) + if(CMAKE_BUILD_TYPE STREQUAL "Coverage") + include(Coverage) + add_executable_coverage(${NAME}) + endif() +endmacro() + +function(add_test_to_suite SUITE NAME) + add_test_executable(${NAME} ${ARGN}) add_test_runner(${SUITE} ${NAME} ${NAME}) get_target_from_suite(${SUITE} TARGET) @@ -76,7 +89,7 @@ ) get_target_from_suite(${SUITE} SUITE_TARGET) - add_executable(${NAME} EXCLUDE_FROM_ALL ${ARG_UNPARSED_ARGUMENTS}) + add_test_executable(${NAME} ${ARG_UNPARSED_ARGUMENTS}) add_dependencies("${SUITE_TARGET}" ${NAME}) foreach(_test_source ${ARG_TESTS}) diff --git a/doc/developer-notes.md b/doc/developer-notes.md --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -283,20 +283,29 @@ ### Compiling for test coverage -LCOV can be used to generate a test coverage report based upon `make check` -execution. LCOV must be installed on your system (e.g. the `lcov` package -on Debian/Ubuntu). +LCOV can be used to generate a test coverage report based upon some test targets +execution. Some packages are required to generate the coverage report: +`c++filt`, `gcov`, `genhtml`, `lcov` and `python3`. -To enable LCOV report generation during test runs: +To install these dependencies on Debian 10: ```shell -./configure --enable-lcov -make -make cov +sudo apt install binutils-common g++ lcov python3 +``` + +To enable LCOV report generation during test runs: -# A coverage report will now be accessible at `./test_bitcoin.coverage/index.html`. +```shell +cmake -GNinja .. -DCMAKE_BUILD_TYPE=Coverage -DCCACHE=OFF +ninja coverage-check-all ``` +A coverage report will now be accessible at `./check_all.coverage/index.html`. + +To include branch coverage, you can add the `-DENABLE_BRANCH_COVERAGE=ON` option +to the `cmake` command line. + + ### Sanitizers Bitcoin ABC can be compiled with various "sanitizers" enabled, which add diff --git a/src/secp256k1/CMakeLists.txt b/src/secp256k1/CMakeLists.txt --- a/src/secp256k1/CMakeLists.txt +++ b/src/secp256k1/CMakeLists.txt @@ -6,14 +6,24 @@ # Add path for custom modules when building as a standalone project list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules) +option(SECP256K1_ENABLE_BRANCH_COVERAGE "Enable branch coverage" OFF) + +if(CMAKE_BUILD_TYPE STREQUAL "Coverage") + include(Coverage) + + if(SECP256K1_ENABLE_BRANCH_COVERAGE) + enable_branch_coverage() + endif() +endif() + # Default to RelWithDebInfo configuration if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Select the configuration for the build" FORCE) endif() -# TODO: use -O0 instead of -O3 when coverage is enabled set(CMAKE_C_FLAGS_RELWITHDEBINFO "-g -O3") +set(CMAKE_C_FLAGS_COVERAGE "-g -O0") include(AddCompilerFlags) @@ -279,12 +289,11 @@ endfunction() create_secp256k1_test(tests src/tests.c) - target_compile_definitions(tests PRIVATE VERIFY) - create_secp256k1_test(exhaustive_tests src/tests_exhaustive.c) + # This should not be enabled at the same time as coverage is. - # TODO: support coverage. - target_compile_definitions(exhaustive_tests PRIVATE VERIFY) + target_compile_definitions(tests PRIVATE $<$>:VERIFY>) + target_compile_definitions(exhaustive_tests PRIVATE $<$>:VERIFY>) if(SECP256K1_ENABLE_JNI) set(SECP256k1_JNI_TEST_JAR "secp256k1-jni-test") diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -60,6 +60,9 @@ USES_TERMINAL VERBATIM ) + if(CMAKE_BUILD_TYPE STREQUAL "Coverage") + add_custom_target_coverage(${TARGET}) + endif() endmacro() add_functional_test_check(check-functional