diff --git a/CMakeLists.txt b/CMakeLists.txt index 334a89d51b..4fa309a30c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,75 +1,119 @@ # Copyright (c) 2017 The Bitcoin developers cmake_minimum_required(VERSION 3.13) set(CMAKE_USER_MAKE_RULES_OVERRIDE "${CMAKE_SOURCE_DIR}/cmake/modules/OverrideInitFlags.cmake" ) project(bitcoin-abc VERSION 0.21.5 DESCRIPTION "Bitcoin ABC is a full node implementation of the Bitcoin Cash protocol." HOMEPAGE_URL "https://www.bitcoinabc.org" ) # Package information set(PACKAGE_NAME "Bitcoin ABC") # Copyright set(COPYRIGHT_YEAR 2020) set(COPYRIGHT_HOLDERS "The %s developers") set(COPYRIGHT_HOLDERS_SUBSTITUTION Bitcoin) string(REPLACE "%s" ${COPYRIGHT_HOLDERS_SUBSTITUTION} COPYRIGHT_HOLDERS_FINAL ${COPYRIGHT_HOLDERS}) # Add path for custom modules list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules) # Make contrib script accessible. set(CONTRIB_PATH ${CMAKE_CURRENT_SOURCE_DIR}/contrib) # Default to RelWithDebInfo configuration if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Select the configuration for the build" FORCE) + set(__NO_USER_CMAKE_BUILD_TYPE ON CACHE BOOL "True if the user didn't set a build type on the command line") endif() # Find the python interpreter. This is required for several targets. find_package(Python 3.5 COMPONENTS Interpreter REQUIRED) # Add the magic targets `check-*` add_custom_target(check-all) add_custom_target(check) add_custom_target(check-extended) add_custom_target(check-upgrade-activated) add_custom_target(check-upgrade-activated-extended) include(PackageHelper) exclude_git_ignored_files_from_source_package() # Ignore hidden files and directories (starting with a '.') set_property(GLOBAL APPEND PROPERTY SOURCE_PACKAGE_IGNORE_FILES "/\\\\.") # If the build is out-of-tree, then the build directory can be ignored. if(NOT CMAKE_BINARY_DIR STREQUAL CMAKE_SOURCE_DIR) set_property(GLOBAL APPEND PROPERTY SOURCE_PACKAGE_IGNORE_FILES "${CMAKE_BINARY_DIR}/" ) endif() exclude_from_source_package( # Subdirectories "arcanist/" "depends/" # Files "[^.]+[.]md$" "Dockerfile-doxygen" ) +option(ENABLE_COVERAGE "Enable coverage" OFF) +option(ENABLE_BRANCH_COVERAGE "Enable branch coverage" OFF) + +if(ENABLE_COVERAGE) + include(Coverage) + enable_coverage(${ENABLE_BRANCH_COVERAGE}) + + include(AddCompilerFlags) + + # If no build type is manually defined, override the optimization level. + # Otherwise, alert the user than the coverage result might be useless. + if(__NO_USER_CMAKE_BUILD_TYPE) + set_c_optimization_level(0) + + # Setting -Og instead of -O0 is a workaround for the GCC bug 90380: + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90380 + # + # This bug is fixed upstream, but is not widely distributed yet. + # Fixed in GCC versions: + # - GCC 7.x: versions <= 7.2 are unaffected + # - GCC 8.x: versions >= 8.3.1 + # - GCC 9.x: versions >= 9.1.1 + # - GCC 10.x: all versions + set_cxx_optimization_level(g) + else() + message(WARNING "It is advised to not enforce CMAKE_BUILD_TYPE to get the best coverage results") + endif() + + exclude_from_coverage( + "depends" + "src/bench" + "src/crypto/ctaes" + "src/leveldb" + "src/univalue" + ) + + add_custom_target_coverage(check) + add_custom_target_coverage(check-all) + add_custom_target_coverage(check-extended) + add_custom_target_coverage(check-upgrade-activated) + add_custom_target_coverage(check-upgrade-activated-extended) +endif() + add_subdirectory(src) add_subdirectory(test) add_subdirectory(contrib) add_subdirectory(doc) include(PackageOptions.cmake) diff --git a/cmake/modules/AddCompilerFlags.cmake b/cmake/modules/AddCompilerFlags.cmake index 3d061354a8..4215f6972c 100644 --- a/cmake/modules/AddCompilerFlags.cmake +++ b/cmake/modules/AddCompilerFlags.cmake @@ -1,143 +1,166 @@ # Allow to easily add flags for C and C++ include(CheckCXXCompilerFlag) include(CheckCCompilerFlag) include(SanitizeHelper) function(check_compiler_flag RESULT LANGUAGE FLAG) sanitize_c_cxx_definition("have_${LANGUAGE}_" ${FLAG} TEST_NAME) if("${LANGUAGE}" STREQUAL "C") CHECK_C_COMPILER_FLAG(${FLAG} ${TEST_NAME}) elseif("${LANGUAGE}" STREQUAL "CXX") CHECK_CXX_COMPILER_FLAG(${FLAG} ${TEST_NAME}) else() message(FATAL_ERROR "check_compiler_flag LANGUAGE should be C or CXX") endif() set(${RESULT} ${${TEST_NAME}} PARENT_SCOPE) endfunction() function(add_compiler_flags_for_language LANGUAGE) foreach(f ${ARGN}) check_compiler_flag(FLAG_IS_SUPPORTED ${LANGUAGE} ${f}) if(${FLAG_IS_SUPPORTED}) add_compile_options($<$:${f}>) endif() endforeach() endfunction() macro(add_c_compiler_flags) add_compiler_flags_for_language(C ${ARGN}) endmacro() macro(add_cxx_compiler_flags) add_compiler_flags_for_language(CXX ${ARGN}) endmacro() macro(add_compiler_flags) add_c_compiler_flags(${ARGN}) add_cxx_compiler_flags(${ARGN}) endmacro() macro(remove_compiler_flags_from_var TARGET) foreach(f ${ARGN}) string(REGEX REPLACE "${f}( |$)" "" ${TARGET} "${${TARGET}}") endforeach() endmacro() function(remove_c_compiler_flags) remove_compiler_flags_from_var(CMAKE_C_FLAGS ${ARGN}) set(CMAKE_C_FLAGS ${CMAKE_C_FLAGS} PARENT_SCOPE) if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "") string(TOUPPER "CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}" BUILD_TYPE_FLAGS) remove_compiler_flags_from_var(${BUILD_TYPE_FLAGS} ${ARGN}) set(${BUILD_TYPE_FLAGS} ${${BUILD_TYPE_FLAGS}} PARENT_SCOPE) endif() endfunction() function(remove_cxx_compiler_flags) remove_compiler_flags_from_var(CMAKE_CXX_FLAGS ${ARGN}) set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} PARENT_SCOPE) if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "") string(TOUPPER "CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}" BUILD_TYPE_FLAGS) remove_compiler_flags_from_var(${BUILD_TYPE_FLAGS} ${ARGN}) set(${BUILD_TYPE_FLAGS} ${${BUILD_TYPE_FLAGS}} PARENT_SCOPE) endif() endfunction() macro(remove_compiler_flags) remove_c_compiler_flags(${ARGN}) remove_cxx_compiler_flags(${ARGN}) endmacro() function(add_compile_options_to_configuration_for_language CONFIGURATION LANGUAGE) foreach(f ${ARGN}) check_compiler_flag(FLAG_IS_SUPPORTED ${LANGUAGE} ${f}) if(${FLAG_IS_SUPPORTED}) add_compile_options($<$,$>:${f}>) endif() endforeach() endfunction() macro(add_c_compile_options_to_configuration CONFIGURATION) add_compile_options_to_configuration_for_language(${CONFIGURATION} C ${ARGN}) endmacro() macro(add_cxx_compile_options_to_configuration CONFIGURATION) add_compile_options_to_configuration_for_language(${CONFIGURATION} CXX ${ARGN}) endmacro() macro(add_compile_options_to_configuration CONFIGURATION) add_c_compile_options_to_configuration(${CONFIGURATION} ${ARGN}) add_cxx_compile_options_to_configuration(${CONFIGURATION} ${ARGN}) endmacro() function(add_compile_definitions_to_configuration CONFIGURATION) foreach(f ${ARGN}) add_compile_definitions($<$:${f}>) endforeach() endfunction() # Note that CMake does not provide any facility to check that a linker flag is # supported by the compiler. # However since CMake 3.2 introduced the CMP0056 policy, the # CMAKE_EXE_LINKER_FLAGS variable is used by the try_compile function, so there # is a workaround that allow for testing the linker flags. function(check_linker_flag RESULT FLAG) # Some linkers (e.g.: Clang) will issue a -Wunused-command-line-argument # warning when an unknown linker flag is set. # Using -Werror will promote these warnings to errors so # CHECK_CXX_COMPILER_FLAG() will return false, preventing the flag from # being set. set(WERROR_UNUSED_ARG -Werror=unused-command-line-argument) check_compiler_flag(IS_WERROR_SUPPORTED CXX ${WERROR_UNUSED_ARG}) if(${IS_WERROR_SUPPORTED}) set(CMAKE_REQUIRED_FLAGS ${WERROR_UNUSED_ARG}) endif() # Save the current linker flags set(SAVE_CMAKE_EXE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS}) # Append the flag under test to the linker flags string(APPEND CMAKE_EXE_LINKER_FLAGS " ${FLAG}") # CHECK_CXX_COMPILER_FLAG calls CHECK_CXX_SOURCE_COMPILES which in turn # calls try_compile, so it will check our flag CHECK_CXX_COMPILER_FLAG("" ${RESULT}) # Restore CMAKE_EXE_LINKER_FLAGS set(CMAKE_EXE_LINKER_FLAGS ${SAVE_CMAKE_EXE_LINKER_FLAGS}) set(${RESULT} ${${RESULT}} PARENT_SCOPE) endfunction() function(add_linker_flags) foreach(f ${ARGN}) sanitize_c_cxx_definition("have_linker_" ${f} FLAG_IS_SUPPORTED) check_linker_flag(${FLAG_IS_SUPPORTED} ${f}) if(${FLAG_IS_SUPPORTED}) add_link_options(${f}) endif() endforeach() endfunction() + +macro(remove_optimization_level_from_var VAR) + string(REGEX REPLACE "-O[0-3gs]( |$)" "" ${VAR} "${${VAR}}") +endmacro() + +function(set_optimization_level_for_language LANGUAGE LEVEL) + if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "") + string(TOUPPER "CMAKE_${LANGUAGE}_FLAGS_${CMAKE_BUILD_TYPE}" BUILD_TYPE_FLAGS) + remove_optimization_level_from_var(${BUILD_TYPE_FLAGS}) + set(${BUILD_TYPE_FLAGS} "${${BUILD_TYPE_FLAGS}}" PARENT_SCOPE) + endif() + + remove_optimization_level_from_var(CMAKE_${LANGUAGE}_FLAGS) + set(CMAKE_${LANGUAGE}_FLAGS "${CMAKE_${LANGUAGE}_FLAGS} -O${LEVEL}" PARENT_SCOPE) +endfunction() + +macro(set_c_optimization_level LEVEL) + set_optimization_level_for_language(C ${LEVEL}) +endmacro() + +macro(set_cxx_optimization_level LEVEL) + set_optimization_level_for_language(CXX ${LEVEL}) +endmacro() diff --git a/cmake/modules/Coverage.cmake b/cmake/modules/Coverage.cmake new file mode 100644 index 0000000000..4dce5e5fbd --- /dev/null +++ b/cmake/modules/Coverage.cmake @@ -0,0 +1,149 @@ +# Copyright (c) 2020 The Bitcoin developers +############################################################################### +# Provide support for building test coverage reports. +# +# Dependencies: +# - c++filt +# - gcov +# - genhtml +# - lcov +# - python3 +# +# The basic workflow for building coverage report is as follow: +# 1/ Build the instrumented code by passing the --coverage flag to the +# compiler. For better accuracy there should be no optimization. +# 2/ Build the code, prior to run any test. Capture an initial state with +# `lcov` to get a baseline for the coverage information (function names, +# number of lines). This is achieved by running the `coverage-baseline` +# target. +# 3/ The coverage data is filtered to remove coverage info from code which is +# not of interest (such as 3rd party libs, generated files, etc.). +# 4/ Run the tests, and capture the generated coverage data with `lcov`. Again, +# the coverage data is filtered. After the test target has run, the +# coverage counters are reset in order to make each test run produce an +# independent output. +# 5/ Combine the test coverage data with the baseline data using `lcov` to +# produce the final coverage information for the test. +# 6/ Use `genhtml` to generate a HTML report from the combined coverage data. +# +# How to use: +# 1/ Enable coverage with the `enable_coverage()` function. The branch coverage +# can be enabled by setting the `ENABLE_BRANCH_COVERAGE` parameter to true. +# 2/ Call `exclude_from_coverage()` to exclude paths from the coverage report. +# 3/ Use `add_custom_target_coverage()` to enable coverage reporting for your +# test target. This generates a new `coverage-` target that will run +# the tests and create the HTML report in a `` directory. +############################################################################### + +include(SanitizeHelper) + +# Exclude directories (and subdirectories) from the coverage report. +function(exclude_from_coverage) + foreach(_dir ${ARGN}) + get_filename_component(_abspath "${_dir}" ABSOLUTE) + set_property(GLOBAL APPEND_STRING PROPERTY LCOV_FILTER_PATTERN " -p ${_abspath}") + endforeach() +endfunction() + +function(enable_coverage ENABLE_BRANCH_COVERAGE) + set(__ENABLE_COVERAGE ON CACHE INTERNAL "Coverage is enabled") + + # Required dependencies. + # c++filt is needed to demangle c++ function names at HTML generation time. + include(DoOrFail) + find_program_or_fail(LCOV_EXECUTABLE lcov) + find_program_or_fail(GCOV_EXECUTABLE gcov) + find_program_or_fail(GENHTML_EXECUTABLE genhtml) + find_program_or_fail(CXXFILT_EXECUTABLE c++filt) + find_package(Python3 COMPONENTS Interpreter REQUIRED) + set(__COVERAGE_PYTHON "${Python3_EXECUTABLE}" CACHE PATH "Path to the Python interpreter") + + get_property(_project_languages GLOBAL PROPERTY ENABLED_LANGUAGES) + set(COVERAGE_FLAG --coverage) + + foreach(_language ${_project_languages}) + sanitize_c_cxx_definition( + "supports_${_language}_" + ${COVERAGE_FLAG} + SUPPORTS_COVERAGE + ) + + set(_save_linker_flags ${CMAKE_EXE_LINKER_FLAGS}) + string(APPEND CMAKE_EXE_LINKER_FLAGS " ${COVERAGE_FLAG}") + + if("${_language}" STREQUAL "C") + include(CheckCCompilerFlag) + CHECK_C_COMPILER_FLAG(${COVERAGE_FLAG} ${SUPPORTS_COVERAGE}) + elseif("${_language}" STREQUAL "CXX") + include(CheckCXXCompilerFlag) + CHECK_CXX_COMPILER_FLAG(${COVERAGE_FLAG} ${SUPPORTS_COVERAGE}) + else() + message(WARNING "Coverage is not supported for the ${_language} language") + endif() + + set(CMAKE_EXE_LINKER_FLAGS ${_save_linker_flags}) + + if(NOT ${SUPPORTS_COVERAGE}) + message(FATAL_ERROR "The ${COVERAGE_FLAG} option is not supported by your ${_language} compiler") + endif() + + add_compile_options($<$:${COVERAGE_FLAG}>) + add_link_options($<$:${COVERAGE_FLAG}>) + endforeach() + + # Exclude some path by default, such as system headers and generated files. + exclude_from_coverage( + "${CMAKE_BINARY_DIR}" + "/usr/include" + "/usr/lib" + "/usr/lib64" + ) + + if(ENABLE_BRANCH_COVERAGE) + string(APPEND LCOV_OPTIONS "--rc lcov_branch_coverage=1") + set(LCOV_OPTIONS "${LCOV_OPTIONS}" CACHE STRING "Lcov options") + endif() +endfunction() + +function(add_custom_target_coverage TARGET) + # Coverage should have been enabled in order to create the new coverage-* + # targets. + if(NOT __ENABLE_COVERAGE) + return() + endif() + + get_property(LCOV_FILTER_PATTERN GLOBAL PROPERTY LCOV_FILTER_PATTERN) + + # Make sure we generate the base coverage data before building this target. + if(NOT TARGET coverage-baseline) + configure_file( + "${CMAKE_SOURCE_DIR}/cmake/templates/CoverageBaseline.sh.in" + "${CMAKE_BINARY_DIR}/CoverageBaseline.sh" + ) + + add_custom_command( + COMMENT "Generating baseline coverage info" + OUTPUT "${CMAKE_BINARY_DIR}/baseline.info" + COMMAND "${CMAKE_BINARY_DIR}/CoverageBaseline.sh" + VERBATIM USES_TERMINAL + ) + + add_custom_target(coverage-baseline + DEPENDS "${CMAKE_BINARY_DIR}/baseline.info" + ) + endif() + + sanitize_c_cxx_definition("" "${TARGET}" SANITIZED_TARGET) + + configure_file( + "${CMAKE_SOURCE_DIR}/cmake/templates/CoverageTest.sh.in" + "${CMAKE_BINARY_DIR}/Coverage-${TARGET}.sh" + ) + + add_custom_target(coverage-${TARGET} + DEPENDS coverage-baseline ${TARGET} + COMMENT "Generating ${TARGET} coverage report" + COMMAND "${CMAKE_BINARY_DIR}/Coverage-${TARGET}.sh" + VERBATIM USES_TERMINAL + ) +endfunction() diff --git a/cmake/modules/TestSuite.cmake b/cmake/modules/TestSuite.cmake index 4f4290fbfd..1d3abb1a34 100644 --- a/cmake/modules/TestSuite.cmake +++ b/cmake/modules/TestSuite.cmake @@ -1,120 +1,124 @@ # Allow to easily build test suites macro(add_test_environment VARIABLE VALUE) set_property(GLOBAL APPEND PROPERTY TEST_ENVIRONMENT "${VARIABLE}=${VALUE}") endmacro() function(add_test_custom_target TARGET) cmake_parse_arguments(ARG "" "" "CUSTOM_TARGET_ARGS;TEST_COMMAND" ${ARGN}) get_property(TEST_ENVIRONMENT GLOBAL PROPERTY TEST_ENVIRONMENT) add_custom_target(${TARGET} ${ARG_CUSTOM_TARGET_ARGS} COMMAND ${CMAKE_COMMAND} -E env ${TEST_ENVIRONMENT} ${ARG_TEST_COMMAND} ) endfunction() # Define a new target property to hold the list of tests associated with a test # suite. This property is named UNIT_TESTS to avoid confusion with the directory # level property TESTS. define_property(TARGET PROPERTY UNIT_TESTS BRIEF_DOCS "List of tests" FULL_DOCS "A list of the tests associated with a test suite" ) macro(get_target_from_suite SUITE TARGET) set(${TARGET} "check-${SUITE}") endmacro() +include(Coverage) + function(create_test_suite_with_parent_targets NAME) get_target_from_suite(${NAME} TARGET) add_custom_target(${TARGET} COMMENT "Running ${NAME} test suite" COMMAND cmake -E echo "PASSED: ${NAME} test suite" ) foreach(PARENT_TARGET ${ARGN}) if(TARGET ${PARENT_TARGET}) add_dependencies(${PARENT_TARGET} ${TARGET}) endif() endforeach() + + add_custom_target_coverage(${TARGET}) endfunction() macro(create_test_suite NAME) create_test_suite_with_parent_targets(${NAME} check-all check-extended) endmacro() set(TEST_RUNNER_TEMPLATE "${CMAKE_CURRENT_LIST_DIR}/../templates/TestRunner.cmake.in") function(add_test_runner SUITE NAME EXECUTABLE) get_target_from_suite(${SUITE} SUITE_TARGET) set(TARGET "${SUITE_TARGET}-${NAME}") add_test_custom_target(${TARGET} TEST_COMMAND "${CMAKE_SOURCE_DIR}/cmake/utils/test_wrapper.sh" "$" "${NAME}.log" ${ARGN} CUSTOM_TARGET_ARGS COMMENT "${SUITE}: testing ${NAME}" DEPENDS ${EXECUTABLE} VERBATIM ) add_dependencies(${SUITE_TARGET} ${TARGET}) endfunction() function(add_test_to_suite SUITE NAME) add_executable(${NAME} EXCLUDE_FROM_ALL ${ARGN}) add_test_runner(${SUITE} ${NAME} ${NAME}) get_target_from_suite(${SUITE} TARGET) set_property( TARGET ${TARGET} APPEND PROPERTY UNIT_TESTS ${NAME} ) endfunction(add_test_to_suite) function(add_boost_unit_tests_to_suite SUITE NAME) cmake_parse_arguments(ARG "" "" "TESTS" ${ARGN} ) get_target_from_suite(${SUITE} SUITE_TARGET) add_executable(${NAME} EXCLUDE_FROM_ALL ${ARG_UNPARSED_ARGUMENTS}) add_dependencies("${SUITE_TARGET}" ${NAME}) foreach(_test_source ${ARG_TESTS}) target_sources(${NAME} PRIVATE "${_test_source}") get_filename_component(_test_name "${_test_source}" NAME_WE) add_test_runner( ${SUITE} ${_test_name} ${NAME} -t "${_test_name}" ) set_property( TARGET ${SUITE_TARGET} APPEND PROPERTY UNIT_TESTS ${_test_name} ) endforeach() find_package(Boost 1.58 REQUIRED unit_test_framework) target_link_libraries(${NAME} Boost::unit_test_framework) # We need to detect if the BOOST_TEST_DYN_LINK flag is required include(CheckCXXSourceCompiles) set(CMAKE_REQUIRED_LIBRARIES Boost::unit_test_framework) check_cxx_source_compiles(" #define BOOST_TEST_DYN_LINK #define BOOST_TEST_MAIN #include " BOOST_TEST_DYN_LINK) if(BOOST_TEST_DYN_LINK) target_compile_definitions(${NAME} PRIVATE BOOST_TEST_DYN_LINK) endif(BOOST_TEST_DYN_LINK) endfunction(add_boost_unit_tests_to_suite) diff --git a/cmake/templates/CoverageBaseline.sh.in b/cmake/templates/CoverageBaseline.sh.in new file mode 100755 index 0000000000..25fcd12e4c --- /dev/null +++ b/cmake/templates/CoverageBaseline.sh.in @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +# Build the default target +"${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" + +# Capture (-c) initial (-i) coverage data in order to get a baseline +# before running any test. +"${LCOV_EXECUTABLE}" --gcov-tool="${GCOV_EXECUTABLE}" ${LCOV_OPTIONS} \ + -c -i -d "${CMAKE_BINARY_DIR}" \ + -o baseline_raw.info + +# Remove the coverage data for the paths matching any of the patterns. +"${__COVERAGE_PYTHON}" "${CMAKE_SOURCE_DIR}/cmake/utils/filter-lcov.py" \ + ${LCOV_FILTER_PATTERN} baseline_raw.info "${CMAKE_BINARY_DIR}/baseline.info" diff --git a/cmake/templates/CoverageTest.sh.in b/cmake/templates/CoverageTest.sh.in new file mode 100755 index 0000000000..79e9d567a3 --- /dev/null +++ b/cmake/templates/CoverageTest.sh.in @@ -0,0 +1,30 @@ +#!/bin/sh + +set -e + +# Capture (-c) coverage data generated by the test. +"${LCOV_EXECUTABLE}" --gcov-tool="${GCOV_EXECUTABLE}" ${LCOV_OPTIONS} \ + -c -d "${CMAKE_BINARY_DIR}" \ + -t ${SANITIZED_TARGET} \ + -o "${TARGET}.info" + +# Reset to zero (-z) the counters (remove the *.gcda coverage files). +"${LCOV_EXECUTABLE}" --gcov-tool="${GCOV_EXECUTABLE}" ${LCOV_OPTIONS} \ + -z -d "${CMAKE_BINARY_DIR}" + +# Remove the coverage data for the paths matching any of the patterns. +"${__COVERAGE_PYTHON}" "${CMAKE_SOURCE_DIR}/cmake/utils/filter-lcov.py" \ + ${LCOV_FILTER_PATTERN} "${TARGET}.info" "${TARGET}_filtered.info" + +# Add (-a) the baseline and test coverage data files to combine them +# into a single one. +"${LCOV_EXECUTABLE}" --gcov-tool="${GCOV_EXECUTABLE}" ${LCOV_OPTIONS} \ + -a "${CMAKE_BINARY_DIR}/baseline.info" \ + -a "${TARGET}_filtered.info" \ + -o "${TARGET}_combined.info" + +# Generate the HTML coverage report from the coverage data. +"${GENHTML_EXECUTABLE}" ${LCOV_OPTIONS} \ + --demangle-cpp \ + -s "${TARGET}_combined.info" \ + -o "${CMAKE_BINARY_DIR}/${TARGET}.coverage" diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 1897b0505c..501d273143 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -1,932 +1,941 @@ Developer Notes =============== **Table of Contents** - [Developer Notes](#developer-notes) - [Coding Style](#coding-style) - [Doxygen comments](#doxygen-comments) - [Development tips and tricks](#development-tips-and-tricks) - [Compiling for debugging](#compiling-for-debugging) - [Compiling for gprof profiling](#compiling-for-gprof-profiling) - [debug.log](#debuglog) - [Writing tests](#writing-tests) - [Writing script integration tests](#writing-script-integration-tests) - [Testnet and Regtest modes](#testnet-and-regtest-modes) - [DEBUG_LOCKORDER](#debug_lockorder) - [Valgrind suppressions file](#valgrind-suppressions-file) - [Compiling for test coverage](#compiling-for-test-coverage) - [Sanitizers](#sanitizers) - [Locking/mutex usage notes](#lockingmutex-usage-notes) - [Threads](#threads) - [Ignoring IDE/editor files](#ignoring-ideeditor-files) - [Development guidelines](#development-guidelines) - [Wallet](#wallet) - [General C++](#general-c) - [C++ data structures](#c-data-structures) - [Strings and formatting](#strings-and-formatting) - [Variable names](#variable-names) - [Threads and synchronization](#threads-and-synchronization) - [Scripts](#scripts) - [Shebang](#shebang) - [Source code organization](#source-code-organization) - [GUI](#gui) - [Unit tests](#unit-tests) - [Third party libraries](#third-party-libraries) - [Git and GitHub tips](#git-and-github-tips) - [RPC interface guidelines](#rpc-interface-guidelines) Coding Style --------------- Various coding styles have been used during the history of the codebase, and the result is not very consistent. However, we're now trying to converge to a single style, so please use it in new code. Old code will be converted gradually and you are encouraged to use the provided [clang-format-diff script](/contrib/devtools/README.md#clang-format-diffpy) to clean up the patch automatically before submitting a pull request. - Basic rules specified in [src/.clang-format](/src/.clang-format). - Braces on new lines for namespaces, classes, functions, methods. - Braces on the same line for everything else. - 4 space indentation (no tabs) for every block except namespaces. - No indentation for `public`/`protected`/`private` or for `namespace`. - No extra spaces inside parenthesis; don't do ( this ) - No space after function names; one space after `if`, `for` and `while`. - Always add braces for block statements (e.g. `if`, `for`, `while`). - `++i` is preferred over `i++`. - `static_assert` is preferred over `assert` where possible. Generally; compile-time checking is preferred over run-time checking. - Use CamelCase for functions/methods, and lowerCamelCase for variables. - GLOBAL_CONSTANTS should use UPPER_SNAKE_CASE. - namespaces should use lower_snake_case. - Function names should generally start with an English command-form verb (e.g. `ValidateTransaction`, `AddTransactionToMempool`, `ConnectBlock`) - Variable names should generally be nouns or past/future tense verbs. (e.g. `canDoThing`, `signatureOperations`, `didThing`) - Avoid using globals, remove existing globals whenever possible. - Class member variable names should be prepended with `m_` - DO choose easily readable identifier names. - DO favor readability over brevity. - DO NOT use Hungarian notation. - DO NOT use abbreviations or contractions within identifiers. - WRONG: mempool - RIGHT: MemoryPool - WRONG: ChangeDir - RIGHT: ChangeDirectory - DO NOT use obscure acronyms, DO uppercase any acronyms. - FINALLY, do not migrate existing code unless refactoring. It makes forwarding-porting from Bitcoin Core more difficult. The naming convention roughly mirrors [Microsoft Naming Conventions](https://docs.microsoft.com/en-us/dotnet/standard/design-guidelines/general-naming-conventions) C++ Coding Standards should strive to follow the [LLVM Coding Standards](https://llvm.org/docs/CodingStandards.html) Code style example: ```c++ // namespaces should be lower_snake_case namespace foo_bar_bob { /** * Class is used for doing classy things. All classes should * have a doxygen comment describing their PURPOSE. That is to say, * why they exist. Functional details can be determined from the code. * @see PerformTask() */ class Class { private: //! memberVariable's name should be lowerCamelCase, and be a noun. int m_memberVariable; public: /** * The documentation before a function or class method should follow Doxygen * spec. The name of the function should start with an english verb which * indicates the intended purpose of this code. * * The function name should be should be CamelCase. * * @param[in] s A description * @param[in] n Another argument description * @pre Precondition for function... */ bool PerformTask(const std::string& s, int n) { // Use lowerChamelCase for local variables. bool didMore = false; // Comment summarizing the intended purpose of this section of code for (int i = 0; i < n; ++i) { if (!DidSomethingFail()) { return false; } ... if (IsSomethingElse()) { DoMore(); didMore = true; } else { DoLess(); } } return didMore; } } } // namespace foo ``` Doxygen comments ----------------- To facilitate the generation of documentation, use doxygen-compatible comment blocks for functions, methods and fields. For example, to describe a function use: ```c++ /** * ... text ... * @param[in] arg1 A description * @param[in] arg2 Another argument description * @pre Precondition for function... */ bool function(int arg1, const char *arg2) ``` A complete list of `@xxx` commands can be found at http://www.stack.nl/~dimitri/doxygen/manual/commands.html. As Doxygen recognizes the comments by the delimiters (`/**` and `*/` in this case), you don't *need* to provide any commands for a comment to be valid; just a description text is fine. To describe a class use the same construct above the class definition: ```c++ /** * Alerts are for notifying old versions if they become too obsolete and * need to upgrade. The message is displayed in the status bar. * @see GetWarnings() */ class CAlert { ``` To describe a member or variable use: ```c++ int var; //!< Detailed description after the member ``` or ```cpp //! Description before the member int var; ``` Also OK: ```c++ /// /// ... text ... /// bool function2(int arg1, const char *arg2) ``` Not OK (used plenty in the current source, but not picked up): ```c++ // // ... text ... // ``` A full list of comment syntaxes picked up by doxygen can be found at http://www.stack.nl/~dimitri/doxygen/manual/docblocks.html, but if possible use one of the above styles. To build doxygen locally to test changes to the Doxyfile or visualize your comments before landing changes: ``` # In the build directory, call: doxygen doc/Doxyfile # output goes to doc/doxygen/html/ ``` Development tips and tricks --------------------------- ### Compiling for debugging Run configure with `--enable-debug` to add additional compiler flags that produce better debugging builds. ### Compiling for gprof profiling Run configure with the `--enable-gprof` option, then make. With `cmake` and `ninja`: ``` cmake -GNinja .. -DENABLE_HARDENING=OFF -DENABLE_PROFIILING=gprof ``` ### debug.log If the code is behaving strangely, take a look in the debug.log file in the data directory; error and debugging messages are written there. The `-debug=...` command-line option controls debugging; running with just `-debug` or `-debug=1` will turn on all categories (and give you a very large debug.log file). The Qt code routes `qDebug()` output to debug.log under category "qt": run with `-debug=qt` to see it. ### Writing tests For details on unit tests, see `unit-tests.md` For details on functional tests, see `functional-tests.md` ### Writing script integration tests Script integration tests are built using `src/test/script_tests.cpp`: 1. Uncomment the line with `#define UPDATE_JSON_TESTS` 2. Add a new TestBuilder to the `script_build` test to cover your test case. 3. `make && ./src/test/test_bitcoin --run_test=script_tests` 4. Copy your newly generated test JSON from `/src/script_tests.json.gen` to `src/test/data/script_tests.json`. Please commit your TestBuilder along with your generated test JSON and cleanup the uncommented #define before code review. ### Testnet and Regtest modes Run with the `-testnet` option to run with "play bitcoins" on the test network, if you are testing multi-machine code that needs to operate across the internet. If you are testing something that can run on one machine, run with the `-regtest` option. In regression test mode, blocks can be created on-demand; see [test/functional/](/test/functional) for tests that run in `-regtest` mode. ### DEBUG_LOCKORDER Bitcoin ABC is a multi-threaded application, and deadlocks or other multi-threading bugs can be very difficult to track down. The `--enable-debug` configure option adds `-DDEBUG_LOCKORDER` to the compiler flags. This inserts run-time checks to keep track of which locks are held, and adds warnings to the debug.log file if inconsistencies are detected. ### Valgrind suppressions file Valgrind is a programming tool for memory debugging, memory leak detection, and profiling. The repo contains a Valgrind suppressions file ([`valgrind.supp`](contrib/valgrind.supp)) which includes known Valgrind warnings in our dependencies that cannot be fixed in-tree. Example use: ```shell $ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ --show-leak-kinds=all src/test/test_bitcoin --log_level=test_suite $ valgrind -v --leak-check=full src/bitcoind -printtoconsole ``` ### Compiling for test coverage -LCOV can be used to generate a test coverage report based upon `make check` -execution. LCOV must be installed on your system (e.g. the `lcov` package -on Debian/Ubuntu). +LCOV can be used to generate a test coverage report based upon some test targets +execution. Some packages are required to generate the coverage report: +`c++filt`, `gcov`, `genhtml`, `lcov` and `python3`. -To enable LCOV report generation during test runs: +To install these dependencies on Debian 10: ```shell -./configure --enable-lcov -make -make cov +sudo apt install binutils-common g++ lcov python3 +``` + +To enable LCOV report generation during test runs: -# A coverage report will now be accessible at `./test_bitcoin.coverage/index.html`. +```shell +cmake -GNinja .. -DENABLE_COVERAGE=ON +ninja coverage-check-all ``` +A coverage report will now be accessible at `./check-all.coverage/index.html`. + +To include branch coverage, you can add the `-DENABLE_BRANCH_COVERAGE=ON` option +to the `cmake` command line. + + ### Sanitizers Bitcoin ABC can be compiled with various "sanitizers" enabled, which add instrumentation for issues regarding things like memory safety, thread race conditions, or undefined behavior. This is controlled with the `--with-sanitizers` configure flag, which should be a comma separated list of sanitizers to enable. The sanitizer list should correspond to supported `-fsanitize=` options in your compiler. These sanitizers have runtime overhead, so they are most useful when testing changes or producing debugging builds. Some examples: ```bash # Enable both the address sanitizer and the undefined behavior sanitizer ./configure --with-sanitizers=address,undefined # Enable the thread sanitizer ./configure --with-sanitizers=thread ``` If you are compiling with GCC you will typically need to install corresponding "san" libraries to actually compile with these flags, e.g. libasan for the address sanitizer, libtsan for the thread sanitizer, and libubsan for the undefined sanitizer. If you are missing required libraries, the configure script will fail with a linker error when testing the sanitizer flags. The test suite should pass cleanly with the `thread` and `undefined` sanitizers, but there are a number of known problems when using the `address` sanitizer. The address sanitizer is known to fail in [sha256_sse4::Transform](/src/crypto/sha256_sse4.cpp) which makes it unusable unless you also use `--disable-asm` when running configure. We would like to fix sanitizer issues, so please send pull requests if you can fix any errors found by the address sanitizer (or any other sanitizer). Not all sanitizer options can be enabled at the same time, e.g. trying to build with `--with-sanitizers=address,thread` will fail in the configure script as these sanitizers are mutually incompatible. Refer to your compiler manual to learn more about these options and which sanitizers are supported by your compiler. Additional resources: * [AddressSanitizer](https://clang.llvm.org/docs/AddressSanitizer.html) * [LeakSanitizer](https://clang.llvm.org/docs/LeakSanitizer.html) * [MemorySanitizer](https://clang.llvm.org/docs/MemorySanitizer.html) * [ThreadSanitizer](https://clang.llvm.org/docs/ThreadSanitizer.html) * [UndefinedBehaviorSanitizer](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html) * [GCC Instrumentation Options](https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html) * [Google Sanitizers Wiki](https://github.com/google/sanitizers/wiki) * [Issue #12691: Enable -fsanitize flags in Travis](https://github.com/bitcoin/bitcoin/issues/12691) Locking/mutex usage notes ------------------------- The code is multi-threaded, and uses mutexes and the `LOCK` and `TRY_LOCK` macros to protect data structures. Deadlocks due to inconsistent lock ordering (thread 1 locks `cs_main` and then `cs_wallet`, while thread 2 locks them in the opposite order: result, deadlock as each waits for the other to release its lock) are a problem. Compile with `-DDEBUG_LOCKORDER` (or use `--enable-debug`) to get lock order inconsistencies reported in the debug.log file. Re-architecting the core code so there are better-defined interfaces between the various components is a goal, with any necessary locking done by the components (e.g. see the self-contained `CBasicKeyStore` class and its `cs_KeyStore` lock for example). Threads ------- - ThreadScriptCheck : Verifies block scripts. - ThreadImport : Loads blocks from blk*.dat files or bootstrap.dat. - StartNode : Starts other threads. - ThreadDNSAddressSeed : Loads addresses of peers from the DNS. - ThreadMapPort : Universal plug-and-play startup/shutdown - ThreadSocketHandler : Sends/Receives data from peers on port 8333. - ThreadOpenAddedConnections : Opens network connections to added nodes. - ThreadOpenConnections : Initiates new connections to peers. - ThreadMessageHandler : Higher-level message handling (sending and receiving). - DumpAddresses : Dumps IP addresses of nodes to peers.dat. - ThreadRPCServer : Remote procedure call handler, listens on port 8332 for connections and services them. - Shutdown : Does an orderly shutdown of everything. Ignoring IDE/editor files -------------------------- In closed-source environments in which everyone uses the same IDE it is common to add temporary files it produces to the project-wide `.gitignore` file. However, in open source software such as Bitcoin Core, where everyone uses their own editors/IDE/tools, it is less common. Only you know what files your editor produces and this may change from version to version. The canonical way to do this is thus to create your local gitignore. Add this to `~/.gitconfig`: ``` [core] excludesfile = /home/.../.gitignore_global ``` (alternatively, type the command `git config --global core.excludesfile ~/.gitignore_global` on a terminal) Then put your favourite tool's temporary filenames in that file, e.g. ``` # NetBeans nbproject/ ``` Another option is to create a per-repository excludes file `.git/info/exclude`. These are not committed but apply only to one repository. If a set of tools is used by the build system or scripts the repository (for example, lcov) it is perfectly acceptable to add its files to `.gitignore` and commit them. Development guidelines ============================ A few non-style-related recommendations for developers, as well as points to pay attention to for reviewers of Bitcoin Core code. Wallet ------- - Make sure that no crashes happen with run-time option `-disablewallet`. - *Rationale*: In RPC code that conditionally uses the wallet (such as `validateaddress`) it is easy to forget that global pointer `pwalletMain` can be NULL. See `test/functional/disablewallet.py` for functional tests exercising the API with `-disablewallet` - Include `db_cxx.h` (BerkeleyDB header) only when `ENABLE_WALLET` is set - *Rationale*: Otherwise compilation of the disable-wallet build will fail in environments without BerkeleyDB General C++ ------------- - Assertions should not have side-effects - *Rationale*: Even though the source code is set to refuse to compile with assertions disabled, having side-effects in assertions is unexpected and makes the code harder to understand - If you use the `.h`, you must link the `.cpp` - *Rationale*: Include files define the interface for the code in implementation files. Including one but not linking the other is confusing. Please avoid that. Moving functions from the `.h` to the `.cpp` should not result in build errors - Use the RAII (Resource Acquisition Is Initialization) paradigm where possible. For example by using `unique_ptr` for allocations in a function. - *Rationale*: This avoids memory and resource leaks, and ensures exception safety - Use `std::make_unique()` to construct objects owned by `unique_ptr`s - *Rationale*: `std::make_unique` is concise and ensures exception safety in complex expressions. C++ data structures -------------------- - Never use the `std::map []` syntax when reading from a map, but instead use `.find()` - *Rationale*: `[]` does an insert (of the default element) if the item doesn't exist in the map yet. This has resulted in memory leaks in the past, as well as race conditions (expecting read-read behavior). Using `[]` is fine for *writing* to a map - Do not compare an iterator from one data structure with an iterator of another data structure (even if of the same type) - *Rationale*: Behavior is undefined. In C++ parlor this means "may reformat the universe", in practice this has resulted in at least one hard-to-debug crash bug - Watch out for out-of-bounds vector access. `&vch[vch.size()]` is illegal, including `&vch[0]` for an empty vector. Use `vch.data()` and `vch.data() + vch.size()` instead. - Vector bounds checking is only enabled in debug mode. Do not rely on it - Make sure that constructors initialize all fields. If this is skipped for a good reason (i.e., optimization on the critical path), add an explicit comment about this - *Rationale*: Ensure determinism by avoiding accidental use of uninitialized values. Also, static analyzers balk about this. - By default, declare single-argument constructors `explicit`. - *Rationale*: This is a precaution to avoid unintended conversions that might arise when single-argument constructors are used as implicit conversion functions. - Use explicitly signed or unsigned `char`s, or even better `uint8_t` and `int8_t`. Do not use bare `char` unless it is to pass to a third-party API. This type can be signed or unsigned depending on the architecture, which can lead to interoperability problems or dangerous conditions such as out-of-bounds array accesses - Prefer explicit constructions over implicit ones that rely on 'magical' C++ behavior - *Rationale*: Easier to understand what is happening, thus easier to spot mistakes, even for those that are not language lawyers - Initialize all non-static class members where they are defined - *Rationale*: Initializing the members in the declaration makes it easy to spot uninitialized ones, and avoids accidentally reading uninitialized memory ```cpp class A { uint32_t m_count{0}; } ``` Strings and formatting ------------------------ - Use `std::string`, avoid C string manipulation functions - *Rationale*: C++ string handling is marginally safer, less scope for buffer overflows and surprises with `\0` characters. Also some C string manipulations tend to act differently depending on platform, or even the user locale - Use `ParseInt32`, `ParseInt64`, `ParseUInt32`, `ParseUInt64`, `ParseDouble` from `utilstrencodings.h` for number parsing - *Rationale*: These functions do overflow checking, and avoid pesky locale issues Variable names -------------- The shadowing warning (`-Wshadow`) is enabled by default. It prevents issues rising from using a different variable with the same name. E.g. in member initializers, prepend `_` to the argument name shadowing the member name: ```c++ class AddressBookPage { Mode m_mode; } AddressBookPage::AddressBookPage(Mode _mode) : m_mode(_mode) ... ``` When using nested cycles, do not name the inner cycle variable the same as in upper cycle etc. Please name variables so that their names do not shadow variables defined in the source code. Threads and synchronization ---------------------------- - Build and run tests with `-DDEBUG_LOCKORDER` to verify that no potential deadlocks are introduced. As of 0.12, this is defined by default when configuring with `--enable-debug` - When using `LOCK`/`TRY_LOCK` be aware that the lock exists in the context of the current scope, so surround the statement and the code that needs the lock with braces OK: ```c++ { TRY_LOCK(cs_vNodes, lockNodes); ... } ``` Wrong: ```c++ TRY_LOCK(cs_vNodes, lockNodes); { ... } ``` Scripts -------------------------- ### Shebang - Use `#!/usr/bin/env bash` instead of obsolete `#!/bin/bash`. - [*Rationale*](https://github.com/dylanaraps/pure-bash-bible#shebang): `#!/bin/bash` assumes it is always installed to /bin/ which can cause issues; `#!/usr/bin/env bash` searches the user's PATH to find the bash binary. OK: ```bash #!/usr/bin/env bash ``` Wrong: ```bash #!/bin/bash ``` Source code organization -------------------------- - Implementation code should go into the `.cpp` file and not the `.h`, unless necessary due to template usage or when performance due to inlining is critical - *Rationale*: Shorter and simpler header files are easier to read, and reduce compile time - Use only the lowercase alphanumerics (`a-z0-9`), underscore (`_`) and hyphen (`-`) in source code filenames. - *Rationale*: `grep`:ing and auto-completing filenames is easier when using a consistent naming pattern. Potential problems when building on case-insensitive filesystems are avoided when using only lowercase characters in source code filenames. - Don't import anything into the global namespace (`using namespace ...`). Use fully specified types such as `std::string`. - *Rationale*: Avoids symbol conflicts - Terminate namespaces with a comment (`// namespace mynamespace`). The comment should be placed on the same line as the brace closing the namespace, e.g. ```c++ namespace mynamespace { ... } // namespace mynamespace namespace { ... } // namespace ``` - *Rationale*: Avoids confusion about the namespace context Header Inclusions ----------------- - Header inclusions should use angle brackets (`#include <>`). The include path should be relative to the `src` folder. e.g.: `#include ` - Native C++ headers should be preferred over C compatibility headers. e.g.: use `` instead of `` - In order to make the code consistent, header files should be included in the following order, with each section separated by a newline: 1. In a .cpp file, the associated .h is in first position. In a test source, this is the header file under test. 2. The project headers. 3. The test headers. 4. The 3rd party libraries headers. Different libraries should be in different sections. 5. The system libraries. All headers should be lexically ordered inside their block. - Use include guards to avoid the problem of double inclusion. The header file `foo/bar.h` should use the include guard identifier `BITCOIN_FOO_BAR_H`, e.g. ```c++ #ifndef BITCOIN_FOO_BAR_H #define BITCOIN_FOO_BAR_H ... #endif // BITCOIN_FOO_BAR_H ``` GUI ----- - Do not display or manipulate dialogs in model code (classes `*Model`) - *Rationale*: Model classes pass through events and data from the core, they should not interact with the user. That's where View classes come in. The converse also holds: try to not directly access core data structures from Views. - Avoid adding slow or blocking code in the GUI thread. In particular do not add new `interface::Node` and `interface::Wallet` method calls, even if they may be fast now, in case they are changed to lock or communicate across processes in the future. Prefer to offload work from the GUI thread to worker threads (see `RPCExecutor` in console code as an example) or take other steps (see https://doc.qt.io/archives/qq/qq27-responsive-guis.html) to keep the GUI responsive. - *Rationale*: Blocking the GUI thread can increase latency, and lead to hangs and deadlocks. Unit Tests ----------- - Test suite naming convention: The Boost test suite in file `src/test/foo_tests.cpp` should be named `foo_tests`. Test suite names must be unique. Third party libraries --------------------- Several parts of the repository are software maintained elsewhere. Changes to these should preferably be sent upstream but bugfixes may also be submitted to Bitcoin ABC so that they can be integrated quickly. Cosmetic changes should be purely taken upstream. Current third party libraries include: - src/leveldb - Upstream at https://github.com/google/leveldb ; Maintained by Google. - **Note**: Follow the instructions in [Upgrading LevelDB](#upgrading-leveldb) when merging upstream changes to Bitcoin ABC. - src/libsecp256k1 - Upstream at https://github.com/bitcoin-core/secp256k1/ ; actively maintained by Bitcoin Core contributors. Bitcoin ABC is using a modified version of libsecp256k1, some changes might be directly submitted to Bitcoin ABC. See the [secp256k1 README](/src/secp256k1/README.md) for details. - src/crypto/ctaes - Upstream at https://github.com/bitcoin-core/ctaes ; maintained by Bitcoin Core contributors. - src/univalue - Upstream at https://github.com/jgarzik/univalue ; maintained by Jeff Garzik. Upgrading LevelDB --------------------- Extra care must be taken when upgrading LevelDB. This section explains issues you must be aware of. ### File Descriptor Counts In most configurations we use the default LevelDB value for `max_open_files`, which is 1000 at the time of this writing. If LevelDB actually uses this many file descriptors it will cause problems with Bitcoin's `select()` loop, because it may cause new sockets to be created where the fd value is >= 1024. For this reason, on 64-bit Unix systems we rely on an internal LevelDB optimization that uses `mmap()` + `close()` to open table files without actually retaining references to the table file descriptors. If you are upgrading LevelDB, you must sanity check the changes to make sure that this assumption remains valid. In addition to reviewing the upstream changes in `env_posix.cc`, you can use `lsof` to check this. For example, on Linux this command will show open `.ldb` file counts: ```bash $ lsof -p $(pidof bitcoind) |\ awk 'BEGIN { fd=0; mem=0; } /ldb$/ { if ($4 == "mem") mem++; else fd++ } END { printf "mem = %s, fd = %s\n", mem, fd}' mem = 119, fd = 0 ``` The `mem` value shows how many files are mmap'ed, and the `fd` value shows you many file descriptors these files are using. You should check that `fd` is a small number (usually 0 on 64-bit hosts). See the notes in the `SetMaxOpenFiles()` function in `dbwrapper.cc` for more details. ### Consensus Compatibility It is possible for LevelDB changes to inadvertently change consensus compatibility between nodes. This happened in Bitcoin 0.8 (when LevelDB was first introduced). When upgrading LevelDB you should review the upstream changes to check for issues affecting consensus compatibility. For example, if LevelDB had a bug that accidentally prevented a key from being returned in an edge case, and that bug was fixed upstream, the bug "fix" would be an incompatible consensus change. In this situation the correct behavior would be to revert the upstream fix before applying the updates to Bitcoin's copy of LevelDB. In general you should be wary of any upstream changes affecting what data is returned from LevelDB queries. Git and GitHub tips --------------------- - Github is not typically the source of truth for pull requests. See CONTRIBUTING.md for instructions on setting up your repo correctly. - Similarly, your git remote origin should be set to: `ssh://vcs@reviews.bitcoinabc.org:2221/source/bitcoin-abc.git` instead of github.com. See CONTRIBUTING.md for details. - For resolving merge/rebase conflicts, it can be useful to enable diff3 style using `git config merge.conflictstyle diff3`. Instead of <<< yours === theirs >>> you will see <<< yours ||| original === theirs >>> This may make it much clearer what caused the conflict. In this style, you can often just look at what changed between *original* and *theirs*, and mechanically apply that to *yours* (or the other way around). - When reviewing patches which change indentation in C++ files, use `git diff -w` and `git show -w`. This makes the diff algorithm ignore whitespace changes. This feature is also available on github.com, by adding `?w=1` at the end of any URL which shows a diff. - When reviewing patches that change symbol names in many places, use `git diff --word-diff`. This will instead of showing the patch as deleted/added *lines*, show deleted/added *words*. - When reviewing patches that move code around, try using `git diff --patience commit~:old/file.cpp commit:new/file/name.cpp`, and ignoring everything except the moved body of code which should show up as neither `+` or `-` lines. In case it was not a pure move, this may even work when combined with the `-w` or `--word-diff` options described above. - When looking at other's pull requests, it may make sense to add the following section to your `.git/config` file: [remote "upstream-pull"] fetch = +refs/pull/*:refs/remotes/upstream-pull/* url = git@github.com:bitcoin/bitcoin.git This will add an `upstream-pull` remote to your git repository, which can be fetched using `git fetch --all` or `git fetch upstream-pull`. Afterwards, you can use `upstream-pull/NUMBER/head` in arguments to `git show`, `git checkout` and anywhere a commit id would be acceptable to see the changes from pull request NUMBER. RPC interface guidelines -------------------------- A few guidelines for introducing and reviewing new RPC interfaces: - Method naming: use consecutive lower-case names such as `getrawtransaction` and `submitblock` - *Rationale*: Consistency with existing interface. - Argument naming: use snake case `fee_delta` (and not, e.g. camel case `feeDelta`) - *Rationale*: Consistency with existing interface. - Use the JSON parser for parsing, don't manually parse integers or strings from arguments unless absolutely necessary. - *Rationale*: Introduces hand-rolled string manipulation code at both the caller and callee sites, which is error prone, and it is easy to get things such as escaping wrong. JSON already supports nested data structures, no need to re-invent the wheel. - *Exception*: AmountFromValue can parse amounts as string. This was introduced because many JSON parsers and formatters hard-code handling decimal numbers as floating point values, resulting in potential loss of precision. This is unacceptable for monetary values. **Always** use `AmountFromValue` and `ValueFromAmount` when inputting or outputting monetary values. The only exceptions to this are `prioritisetransaction` and `getblocktemplate` because their interface is specified as-is in BIP22. - Missing arguments and 'null' should be treated the same: as default values. If there is no default value, both cases should fail in the same way. The easiest way to follow this guideline is detect unspecified arguments with `params[x].isNull()` instead of `params.size() <= x`. The former returns true if the argument is either null or missing, while the latter returns true if is missing, and false if it is null. - *Rationale*: Avoids surprises when switching to name-based arguments. Missing name-based arguments are passed as 'null'. - Try not to overload methods on argument type. E.g. don't make `getblock(true)` and `getblock("hash")` do different things. - *Rationale*: This is impossible to use with `bitcoin-cli`, and can be surprising to users. - *Exception*: Some RPC calls can take both an `int` and `bool`, most notably when a bool was switched to a multi-value, or due to other historical reasons. **Always** have false map to 0 and true to 1 in this case. - Don't forget to fill in the argument names correctly in the RPC command table. - *Rationale*: If not, the call can not be used with name-based arguments. - Set okSafeMode in the RPC command table to a sensible value: safe mode is when the blockchain is regarded to be in a confused state, and the client deems it unsafe to do anything irreversible such as send. Anything that just queries should be permitted. - *Rationale*: Troubleshooting a node in safe mode is difficult if half the RPCs don't work. - Add every non-string RPC argument `(method, idx, name)` to the table `vRPCConvertParams` in `rpc/client.cpp`. - *Rationale*: `bitcoin-cli` and the GUI debug console use this table to determine how to convert a plaintext command line to JSON. If the types don't match, the method can be unusable from there. - A RPC method must either be a wallet method or a non-wallet method. Do not introduce new methods such as `signrawtransaction` that differ in behavior based on presence of a wallet. - *Rationale*: as well as complicating the implementation and interfering with the introduction of multi-wallet, wallet and non-wallet code should be separated to avoid introducing circular dependencies between code units. - Try to make the RPC response a JSON object. - *Rationale*: If a RPC response is not a JSON object then it is harder to avoid API breakage if new data in the response is needed. - Wallet RPCs call BlockUntilSyncedToCurrentChain to maintain consistency with `getblockchaininfo`'s state immediately prior to the call's execution. Wallet RPCs whose behavior does *not* depend on the current chainstate may omit this call. - *Rationale*: In previous versions of Bitcoin Core, the wallet was always in-sync with the chainstate (by virtue of them all being updated in the same cs_main lock). In order to maintain the behavior that wallet RPCs return results as of at least the highest best-known block an RPC client may be aware of prior to entering a wallet RPC call, we must block until the wallet is caught up to the chainstate as of the RPC call's entry. This also makes the API much easier for RPC clients to reason about. - Be aware of RPC method aliases and generally avoid registering the same callback function pointer for different RPCs. - *Rationale*: RPC methods registered with the same function pointer will be considered aliases and only the first method name will show up in the `help` rpc command list. - *Exception*: Using RPC method aliases may be appropriate in cases where a new RPC is replacing a deprecated RPC, to avoid both RPCs confusingly showing up in the command list. diff --git a/src/secp256k1/CMakeLists.txt b/src/secp256k1/CMakeLists.txt index fadc082c20..0886953c06 100644 --- a/src/secp256k1/CMakeLists.txt +++ b/src/secp256k1/CMakeLists.txt @@ -1,389 +1,415 @@ # Copyright (c) 2017 The Bitcoin developers cmake_minimum_required(VERSION 3.13) project(secp256k1 LANGUAGES C VERSION 0.1.0) # Add path for custom modules when building as a standalone project list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules) # Default to RelWithDebInfo configuration if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Select the configuration for the build" FORCE) + set(__NO_USER_CMAKE_BUILD_TYPE ON CACHE BOOL "True if the user didn't set a build type on the command line") endif() -# TODO: use -O0 instead of -O3 when coverage is enabled set(CMAKE_C_FLAGS_RELWITHDEBINFO "-g -O3") +option(SECP256K1_ENABLE_COVERAGE "Enable coverage" OFF) +option(SECP256K1_ENABLE_BRANCH_COVERAGE "Enable branch coverage" OFF) + include(AddCompilerFlags) +if(SECP256K1_ENABLE_COVERAGE) + include(Coverage) + + enable_coverage(${SECP256K1_ENABLE_BRANCH_COVERAGE}) + + exclude_from_coverage("${CMAKE_CURRENT_SOURCE_DIR}/src/bench") + + # If no build type is manually defined, override the optimization level. + # Otherwise, alert the user than the coverage result might be useless. + if(__NO_USER_CMAKE_BUILD_TYPE) + set_c_optimization_level(0) + else() + message(WARNING "It is advised to not enforce CMAKE_BUILD_TYPE to get the best coverage results") + endif() + + set(COVERAGE 1) +endif() + + + # libsecp256k1 use a different set of flags. add_c_compiler_flags( -pedantic -Wall -Wextra -Wcast-align -Wshadow -Wno-unused-function -Wno-overlength-strings -std=c89 -Wnested-externs -Wstrict-prototypes -Wno-long-long ) # Default visibility is hidden on all targets. set(CMAKE_C_VISIBILITY_PRESET hidden) include_directories( . src # For the config ${CMAKE_CURRENT_BINARY_DIR}/src ) # The library add_library(secp256k1 src/secp256k1.c) target_include_directories(secp256k1 PUBLIC include) set(SECP256K1_PUBLIC_HEADERS include/secp256k1.h include/secp256k1_preallocated.h ) option(SECP256K1_ENABLE_BIGNUM "Use the GMP bignum implementation" OFF) if(SECP256K1_ENABLE_BIGNUM) # We need to link in GMP find_package(GMP REQUIRED) target_link_libraries(secp256k1 GMP::gmp) set(USE_NUM_GMP 1) set(USE_FIELD_INV_NUM 1) set(USE_SCALAR_INV_NUM 1) else() set(USE_NUM_NONE 1) set(USE_FIELD_INV_BUILTIN 1) set(USE_SCALAR_INV_BUILTIN 1) endif() # Guess the target architecture, within the ones with supported ASM. # First check if the CMAKE_C_COMPILER_TARGET is set (should be when # cross compiling), then CMAKE_SYSTEM_PROCESSOR as a fallback if meaningful # (this is not the case for ARM as the content is highly non standard). if(CMAKE_C_COMPILER_TARGET MATCHES "x86_64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") set(SECP256K1_ASM_BUILD_TARGET "x86_64") set(SECP256K1_DEFAULT_USE_ASM ON) elseif(CMAKE_C_COMPILER_TARGET MATCHES "arm-linux-gnueabihf") set(SECP256K1_ASM_BUILD_TARGET "arm-linux-gnueabihf") set(SECP256K1_DEFAULT_USE_ASM ON) endif() # Enable ASM by default only if we are building for a compatible target. # The user can still enable/disable it manually if needed. option(SECP256K1_USE_ASM "Use assembly" ${SECP256K1_DEFAULT_USE_ASM}) if(SECP256K1_USE_ASM) macro(unsupported_asm_error) message(FATAL_ERROR "Assembly is enabled, but not supported for your target architecture." "Re-run cmake with -DSECP256K1_USE_ASM=OFF to disable ASM support." ) endmacro() if(SECP256K1_ASM_BUILD_TARGET MATCHES "x86_64") # We check if amd64 asm is supported. check_c_source_compiles(" #include int main() { uint64_t a = 11, tmp; __asm__ __volatile__(\"movq \$0x100000000,%1; mulq %%rsi\" : \"+a\"(a) : \"S\"(tmp) : \"cc\", \"%rdx\"); return 0; } " USE_ASM_X86_64) if(NOT USE_ASM_X86_64) unsupported_asm_error() endif() elseif(SECP256K1_ASM_BUILD_TARGET MATCHES "arm-linux-gnueabihf") enable_language(ASM) set(USE_EXTERNAL_ASM 1) add_library(secp256k1_common src/asm/field_10x26_arm.s) target_link_libraries(secp256k1 secp256k1_common) else() unsupported_asm_error() endif() endif() # We make sure __int128 is defined include(CheckTypeSize) check_type_size(__int128 SIZEOF___INT128) if(SIZEOF___INT128 EQUAL 16) set(HAVE___INT128 1) else() # If we do not support __int128, we should be falling back # on 32bits implementations for field and scalar. endif() # Select the finite field implementation to use. # This can be autodetected or forced by setting USE_FIELD to 32bit or 64bit. # See the truth table below: # +----------------------------------------------------------------------------------------------------------------------+ # | USE_FIELD=64bit | USE_FIELD=32bit | HAVE___INT128 | USE_ASM_X86_64 | USE_FIELD_5X52 | USE_FIELD_10x26 | Config error | # +----------------------------------------------------------------------------------------------------------------------+ # | 0 | 0 | 0 | 0 | 0 | 1 | 0 | # | 0 | 0 | 0 | 1 | 1 | 0 | 0 | # | 0 | 0 | 1 | 0 | 1 | 0 | 0 | # | 0 | 0 | 1 | 1 | 1 | 0 | 0 | # | 0 | 1 | 0 | 0 | 0 | 1 | 0 | # | 0 | 1 | 0 | 1 | 0 | 1 | 0 | # | 0 | 1 | 1 | 0 | 0 | 1 | 0 | # | 0 | 1 | 1 | 1 | 0 | 1 | 0 | # | 1 | 0 | 0 | 0 | 0 | 0 | 1 | # | 1 | 0 | 0 | 1 | 1 | 0 | 0 | # | 1 | 0 | 1 | 0 | 1 | 0 | 0 | # | 1 | 0 | 1 | 1 | 1 | 0 | 0 | # +----------------------------------------------------------------------------------------------------------------------+ set(USE_FIELD "" CACHE STRING "Force the finite field implementation to use (can be 32bit or 64bit)") if(USE_FIELD STREQUAL "64bit" AND NOT (HAVE___INT128 OR USE_ASM_X86_64)) message(SEND_ERROR "64 finite field requested but the compiler does not support __int128 or inline assembly") elseif(NOT USE_FIELD STREQUAL "32bit" AND (HAVE___INT128 OR USE_ASM_X86_64)) set(USE_FIELD_5X52 1) else() set(USE_FIELD_10X26 1) endif() # Select the scalar implementation to use. # This can be autodetected or forced by setting USE_SCALAR to 32bit or 64bit. # See the truth table below: # +--------------------------------------------------------------------------------------------------------+ # | USE_SCALAR=64bit | USE_SCALAR=32bit | HAVE___INT128 | USE_SCALAR_4X64 | USE_SCALAR_8X32 | Config error | # +--------------------------------------------------------------------------------------------------------+ # | 0 | 0 | 0 | 0 | 1 | 0 | # | 0 | 0 | 1 | 1 | 0 | 0 | # | 0 | 1 | 0 | 0 | 1 | 0 | # | 0 | 1 | 1 | 0 | 1 | 0 | # | 1 | 0 | 0 | 0 | 0 | 1 | # | 1 | 0 | 1 | 1 | 0 | 0 | # +--------------------------------------------------------------------------------------------------------+ set(USE_SCALAR "" CACHE STRING "Force the scalar implementation to use (can be 32bit or 64bit)") if(USE_SCALAR STREQUAL "64bit" AND NOT HAVE___INT128) message(SEND_ERROR "64 scalar requested but the compiler does not support __int128") elseif(NOT USE_SCALAR STREQUAL "32bit" AND HAVE___INT128) set(USE_SCALAR_4X64 1) else() set(USE_SCALAR_8X32 1) endif() option(SECP256K1_BUILD_TEST "Build secp256k1's unit tests" ON) include(CMakeDependentOption) cmake_dependent_option( SECP256K1_BUILD_OPENSSL_TESTS "Build the OpenSSL tests" ON SECP256K1_BUILD_TEST ON ) if(SECP256K1_BUILD_OPENSSL_TESTS) find_package(OpenSSL COMPONENTS Crypto) if(NOT OpenSSL_FOUND) message(FATAL_ERROR "OpenSSL is not found, but is required for some tests. You can disable them by passing -DSECP256K1_BUILD_OPENSSL_TESTS=OFF." ) endif() set(ENABLE_OPENSSL_TESTS 1) endif() # Executable internal to secp256k1 need to have the HAVE_CONFIG_H define set. # For convenience, we wrap this into a function. function(link_secp256k1_internal NAME) target_link_libraries(${NAME} secp256k1) target_compile_definitions(${NAME} PRIVATE HAVE_CONFIG_H SECP256K1_BUILD) if(SECP256K1_BUILD_OPENSSL_TESTS) target_link_libraries(${NAME} OpenSSL::Crypto) endif() endfunction(link_secp256k1_internal) # Phony target to build benchmarks add_custom_target(bench-secp256k1) function(add_secp256k1_bench NAME) set(EXECUTABLE_NAME "${NAME}-bench") add_executable(${EXECUTABLE_NAME} ${ARGN}) link_secp256k1_internal(${EXECUTABLE_NAME}) set(BENCH_NAME "bench-secp256k1-${NAME}") add_custom_target(${BENCH_NAME} COMMENT "Benchmarking libsecp256k1 ${NAME}" COMMAND ${EXECUTABLE_NAME} USES_TERMINAL ) add_dependencies(bench-secp256k1 ${BENCH_NAME}) endfunction(add_secp256k1_bench) # ECDH module option(SECP256K1_ENABLE_MODULE_ECDH "Build libsecp256k1's ECDH module" OFF) if(SECP256K1_ENABLE_MODULE_ECDH) set(ENABLE_MODULE_ECDH 1) add_secp256k1_bench(ecdh src/bench_ecdh.c) list(APPEND SECP256K1_PUBLIC_HEADERS include/secp256k1_ecdh.h) endif() # MultiSet module option(SECP256K1_ENABLE_MODULE_MULTISET "Build libsecp256k1's MULTISET module" OFF) if(SECP256K1_ENABLE_MODULE_MULTISET) set(ENABLE_MODULE_MULTISET 1) add_secp256k1_bench(multiset src/bench_multiset.c) list(APPEND SECP256K1_PUBLIC_HEADERS include/secp256k1_multiset.h) endif() # Recovery module option(SECP256K1_ENABLE_MODULE_RECOVERY "Build libsecp256k1's recovery module" ON) if(SECP256K1_ENABLE_MODULE_RECOVERY) set(ENABLE_MODULE_RECOVERY 1) add_secp256k1_bench(recover src/bench_recover.c) list(APPEND SECP256K1_PUBLIC_HEADERS include/secp256k1_recovery.h) endif() # Schnorr module option(SECP256K1_ENABLE_MODULE_SCHNORR "Build libsecp256k1's Schnorr module" ON) if(SECP256K1_ENABLE_MODULE_SCHNORR) set(ENABLE_MODULE_SCHNORR 1) list(APPEND SECP256K1_PUBLIC_HEADERS include/secp256k1_schnorr.h) endif() # External default callbacks option(SECP256K1_ENABLE_EXTERNAL_DEFAULT_CALLBACKS "Enable external default callbacks" OFF) if(SECP256K1_ENABLE_EXTERNAL_DEFAULT_CALLBACKS) set(USE_EXTERNAL_DEFAULT_CALLBACKS 1) endif() # Endomorphism option(SECP256K1_ENABLE_ENDOMORPHISM "Enable endomorphism" OFF) if(SECP256K1_ENABLE_ENDOMORPHISM) set(USE_ENDOMORPHISM 1) endif() # Make the emult window size customizable. set(SECP256K1_ECMULT_WINDOW_SIZE 15 CACHE STRING "Window size for ecmult precomputation for verification, specified as integer in range [2..24].") if(${SECP256K1_ECMULT_WINDOW_SIZE} LESS 2 OR ${SECP256K1_ECMULT_WINDOW_SIZE} GREATER 24) message(FATAL_ERROR "SECP256K1_ECMULT_WINDOW_SIZE must be an integer in range [2..24]") endif() set(SECP256K1_ECMULT_GEN_PRECISION 4 CACHE STRING "Precision bits to tune the precomputed table size for signing.") set(VALID_PRECISIONS 2 4 8) if(NOT ${SECP256K1_ECMULT_GEN_PRECISION} IN_LIST VALID_PRECISIONS) message(FATAL_ERROR "SECP256K1_ECMULT_GEN_PRECISION not 2, 4, 8") endif() # Static precomputation for elliptic curve multiplication option(SECP256K1_ECMULT_STATIC_PRECOMPUTATION "Precompute libsecp256k1's elliptic curve multiplication tables" ON) if(SECP256K1_ECMULT_STATIC_PRECOMPUTATION) set(USE_ECMULT_STATIC_PRECOMPUTATION 1) include(NativeExecutable) native_add_cmake_flags( "-DSECP256K1_ECMULT_WINDOW_SIZE=${SECP256K1_ECMULT_WINDOW_SIZE}" "-DSECP256K1_ECMULT_GEN_PRECISION=${SECP256K1_ECMULT_GEN_PRECISION}" "-DSECP256K1_USE_ASM=OFF" ) add_native_executable(gen_context src/gen_context.c) add_custom_command( OUTPUT src/ecmult_static_context.h COMMAND gen_context ) target_sources(secp256k1 PRIVATE src/ecmult_static_context.h) endif() include(InstallationHelper) if(BUILD_SHARED_LIBS) install_shared_library(secp256k1 PUBLIC_HEADER ${SECP256K1_PUBLIC_HEADERS}) else() set_property(TARGET secp256k1 PROPERTY PUBLIC_HEADER ${SECP256K1_PUBLIC_HEADERS}) install_target(secp256k1) endif() # Generate the config configure_file(src/libsecp256k1-config.h.cmake.in src/libsecp256k1-config.h ESCAPE_QUOTES) target_compile_definitions(secp256k1 PRIVATE HAVE_CONFIG_H SECP256K1_BUILD) # Build the Java binding option(SECP256K1_ENABLE_JNI "Enable the Java Native Interface binding" OFF) if(SECP256K1_ENABLE_JNI) if(NOT SECP256K1_ENABLE_MODULE_ECDH) message(FATAL_ERROR "The secp256k1 JNI support requires ECDH. Try again with -DSECP256K1_ENABLE_MODULE_ECDH=ON.") endif() find_package(Java REQUIRED) find_package(JNI REQUIRED) include(UseJava) add_library(secp256k1_jni SHARED src/java/org_bitcoin_NativeSecp256k1.c src/java/org_bitcoin_Secp256k1Context.c ) install_shared_library(secp256k1_jni) target_include_directories(secp256k1_jni PUBLIC ${JNI_INCLUDE_DIRS}) # As per CMake documentation: the POSITION_INDEPENDENT_CODE property is set # when a target is created. It defaults to True for SHARED or MODULE library # targets and False otherwise. # The secp256ki_jni library being shared, the property is set and it will # build with PIC enabled. But the secp256k1 dependency might not have the # property set, so it's associated source files won't be built with PIC # enabled. That would cause the linker to fail. # Forcing the property for the secp256k1 library fixes the issue. set_target_properties(secp256k1 PROPERTIES POSITION_INDEPENDENT_CODE ON) link_secp256k1_internal(secp256k1_jni) endif() # Tests if(SECP256K1_BUILD_TEST) include(TestSuite) create_test_suite(secp256k1) function(create_secp256k1_test NAME FILES) add_test_to_suite(secp256k1 ${NAME} EXCLUDE_FROM_ALL ${FILES}) link_secp256k1_internal(${NAME}) endfunction() create_secp256k1_test(tests src/tests.c) - target_compile_definitions(tests PRIVATE VERIFY) - create_secp256k1_test(exhaustive_tests src/tests_exhaustive.c) + # This should not be enabled at the same time as coverage is. - # TODO: support coverage. - target_compile_definitions(exhaustive_tests PRIVATE VERIFY) + # The VERIFY failure branch is not expected to be reached, so it would make + # coverage appear lower if set. + if(NOT SECP256K1_ENABLE_COVERAGE) + target_compile_definitions(tests PRIVATE VERIFY) + target_compile_definitions(exhaustive_tests PRIVATE VERIFY) + endif() if(SECP256K1_ENABLE_JNI) set(SECP256k1_JNI_TEST_JAR "secp256k1-jni-test") set(CMAKE_JNI_TARGET TRUE) add_jar(secp256k1-jni-test-jar SOURCES src/java/org/bitcoin/NativeSecp256k1.java src/java/org/bitcoin/NativeSecp256k1Test.java src/java/org/bitcoin/NativeSecp256k1Util.java src/java/org/bitcoin/Secp256k1Context.java ENTRY_POINT org/bitcoin/NativeSecp256k1Test OUTPUT_NAME "${SECP256k1_JNI_TEST_JAR}" ) add_dependencies(secp256k1-jni-test-jar secp256k1_jni) add_custom_target(check-secp256k1-java COMMAND "${Java_JAVA_EXECUTABLE}" "-Djava.library.path=${CMAKE_CURRENT_BINARY_DIR}" "-jar" "${SECP256k1_JNI_TEST_JAR}.jar" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" ) add_dependencies(check-secp256k1-java secp256k1-jni-test-jar) add_dependencies(check-secp256k1 check-secp256k1-java) endif() endif(SECP256K1_BUILD_TEST) # Benchmarks add_secp256k1_bench(verify src/bench_verify.c) add_secp256k1_bench(sign src/bench_sign.c) add_secp256k1_bench(internal src/bench_internal.c) add_secp256k1_bench(ecmult src/bench_ecmult.c) diff --git a/src/secp256k1/README.md b/src/secp256k1/README.md index f8c2029d5b..00a7a1db1c 100644 --- a/src/secp256k1/README.md +++ b/src/secp256k1/README.md @@ -1,123 +1,146 @@ libsecp256k1 ============ [![Build Status](https://travis-ci.org/bitcoin-abc/secp256k1.svg?branch=master)](https://travis-ci.org/bitcoin-abc/secp256k1) Optimized C library for cryptographic operations on curve secp256k1. This library is used for consensus critical cryptographic operations on the Bitcoin Cash network. It is maintained within the Bitcoin ABC repository, and is mirrored as a separate repository for ease of reuse in other Bitcoin Cash projects. Developers who want to contribute may do so at [reviews.bitcoinabc.org](https://reviews.bitcoinabc.org/). Use at your own risk. This library is intended to be the highest quality publicly available library for cryptography on the secp256k1 curve. However, the primary focus of its development has been for usage in the Bitcoin Cash system and usage unlike Bitcoin's may be less well tested, verified, or suffer from a less well thought out interface. Correct usage requires some care and consideration that the library is fit for your application's purpose. Features: * secp256k1 ECDSA signing/verification and key generation. * secp256k1 Schnorr signing/verification ([Bitcoin Cash Schnorr variant](https://www.bitcoincash.org/spec/2019-05-15-schnorr.html)). * Additive and multiplicative tweaking of secret/public keys. * Serialization/parsing of secret keys, public keys, signatures. * Constant time, constant memory access signing and pubkey generation. * Derandomized ECDSA (via RFC6979 or with a caller provided function.) * Very efficient implementation. * Suitable for embedded systems. * Optional module for public key recovery. * Optional module for ECDH key exchange (experimental). * Optional module for multiset hash (experimental). Experimental features have not received enough scrutiny to satisfy the standard of quality of this library but are made available for testing and review by the community. The APIs of these features should not be considered stable. Implementation details ---------------------- * General * No runtime heap allocation. * Extensive testing infrastructure. * Structured to facilitate review and analysis. * Intended to be portable to any system with a C89 compiler and uint64_t support. * No use of floating types. * Expose only higher level interfaces to minimize the API surface and improve application security. ("Be difficult to use insecurely.") * Field operations * Optimized implementation of arithmetic modulo the curve's field size (2^256 - 0x1000003D1). * Using 5 52-bit limbs (including hand-optimized assembly for x86_64, by Diederik Huys). * Using 10 26-bit limbs (including hand-optimized assembly for 32-bit ARM, by Wladimir J. van der Laan). * Field inverses and square roots using a sliding window over blocks of 1s (by Peter Dettman). * Scalar operations * Optimized implementation without data-dependent branches of arithmetic modulo the curve's order. * Using 4 64-bit limbs (relying on __int128 support in the compiler). * Using 8 32-bit limbs. * Group operations * Point addition formula specifically simplified for the curve equation (y^2 = x^3 + 7). * Use addition between points in Jacobian and affine coordinates where possible. * Use a unified addition/doubling formula where necessary to avoid data-dependent branches. * Point/x comparison without a field inversion by comparison in the Jacobian coordinate space. * Point multiplication for verification (a*P + b*G). * Use wNAF notation for point multiplicands. * Use a much larger window for multiples of G, using precomputed multiples. * Use Shamir's trick to do the multiplication with the public key and the generator simultaneously. * Optionally (off by default) use secp256k1's efficiently-computable endomorphism to split the P multiplicand into 2 half-sized ones. * Point multiplication for signing * Use a precomputed table of multiples of powers of 16 multiplied with the generator, so general multiplication becomes a series of additions. * Intended to be completely free of timing sidechannels for secret-key operations (on reasonable hardware/toolchains) * Access the table with branch-free conditional moves so memory access is uniform. * No data-dependent branches * Optional runtime blinding which attempts to frustrate differential power analysis. * The precomputed tables add and eventually subtract points for which no known scalar (secret key) is known, preventing even an attacker with control over the secret key used to control the data internally. Build steps ----------- libsecp256k1 can be built using autotools: ```bash ./autogen.sh mkdir build cd build ../configure make make check sudo make install # optional ``` Or using CMake: ```bash mkdir build cd build cmake -GNinja .. ninja ninja check-secp256k1 sudo ninja install # optional ``` Exhaustive tests ----------- $ ./exhaustive_tests With valgrind, you might need to increase the max stack size: $ valgrind --max-stackframe=2500000 ./exhaustive_tests Test coverage ----------- This library aims to have full coverage of the reachable lines and branches. -To create a test coverage report, configure with `--enable-coverage` (use of GCC is necessary): +__To create a test coverage report with autotools:__ + +Configure with `--enable-coverage` (use of GCC is necessary): $ ./configure --enable-coverage Run the tests: $ make check To create a report, `gcovr` is recommended, as it includes branch coverage reporting: $ gcovr --exclude 'src/bench*' --print-summary To create a HTML report with coloured and annotated source code: $ gcovr --exclude 'src/bench*' --html --html-details -o coverage.html + +__To create a test coverage report with CMake:__ + +Make sure you installed the dependencies first, and they are in your `PATH`: +`c++filt`, `gcov`, `genhtml`, `lcov` and `python3`. + +Then run the build, tests and generate the coverage report with: + +```bash +mkdir coverage +cd coverage +cmake -GNinja .. \ + -DCMAKE_C_COMPILER=gcc \ + -DSECP256K1_ENABLE_COVERAGE=ON \ + -DSECP256K1_ENABLE_BRANCH_COVERAGE=ON # optional +ninja coverage-check-secp256k1 +``` + +The coverage report will be available by opening the file +`check-secp256k1.coverage/index.html` with a web browser. + Reporting a vulnerability ------------ See [SECURITY.md](SECURITY.md) diff --git a/src/secp256k1/src/libsecp256k1-config.h.cmake.in b/src/secp256k1/src/libsecp256k1-config.h.cmake.in index 3f3d9de639..54934fe155 100644 --- a/src/secp256k1/src/libsecp256k1-config.h.cmake.in +++ b/src/secp256k1/src/libsecp256k1-config.h.cmake.in @@ -1,39 +1,41 @@ /* Copyright (c) 2017 The Bitcoin developers */ #ifndef LIBSECP256K1_CONFIG_H #define LIBSECP256K1_CONFIG_H #cmakedefine HAVE___INT128 #cmakedefine USE_NUM_GMP #cmakedefine USE_FIELD_INV_NUM #cmakedefine USE_SCALAR_INV_NUM #cmakedefine USE_NUM_NONE #cmakedefine USE_FIELD_INV_BUILTIN #cmakedefine USE_SCALAR_INV_BUILTIN #cmakedefine USE_SCALAR_4X64 #cmakedefine USE_FIELD_5X52 #cmakedefine USE_SCALAR_8X32 #cmakedefine USE_FIELD_10X26 #cmakedefine USE_ASM_X86_64 #cmakedefine USE_EXTERNAL_ASM #cmakedefine USE_ENDOMORPHISM #cmakedefine USE_EXTERNAL_DEFAULT_CALLBACKS #cmakedefine USE_ECMULT_STATIC_PRECOMPUTATION #define ECMULT_WINDOW_SIZE ${SECP256K1_ECMULT_WINDOW_SIZE} #define ECMULT_GEN_PREC_BITS ${SECP256K1_ECMULT_GEN_PRECISION} #cmakedefine ENABLE_MODULE_ECDH #cmakedefine ENABLE_MODULE_MULTISET #cmakedefine ENABLE_MODULE_RECOVERY #cmakedefine ENABLE_MODULE_SCHNORR #cmakedefine ENABLE_OPENSSL_TESTS +#cmakedefine COVERAGE + #endif /* LIBSECP256K1_CONFIG_H */ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 948ebc1e1d..6a6a1e7f5e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,136 +1,139 @@ ### # Create config.ini file for tests ### set(abs_top_srcdir ${CMAKE_SOURCE_DIR}) set(abs_top_builddir ${CMAKE_BINARY_DIR}) if(CMAKE_SYSTEM_NAME MATCHES "Windows") set(EXEEXT ".exe") endif() if(NOT BUILD_BITCOIN_WALLET) set(ENABLE_WALLET_TRUE "#") endif() if(NOT BUILD_BITCOIN_TX OR NOT BUILD_BITCOIN_TX) set(BUILD_BITCOIN_UTILS_TRUE "#") endif() if(NOT BUILD_BITCOIN_ZMQ) set(ENABLE_ZMQ_TRUE "#") endif() if(NOT "fuzzer" IN_LIST ENABLE_SANITIZERS) set(ENABLE_FUZZ_TRUE "#") endif() # Create build ini file configure_file(config.ini.in config.ini @ONLY) ### # Setup symlinks for testing ### include(SanitizeHelper) function(make_link file) set(src "${CMAKE_CURRENT_SOURCE_DIR}/${file}") set(dest "${CMAKE_CURRENT_BINARY_DIR}/${file}") # Create the target directory and parents if needed. get_filename_component(dest_dir "${dest}" DIRECTORY) file(MAKE_DIRECTORY "${dest_dir}") add_custom_command( OUTPUT "${dest}" COMMAND ${CMAKE_COMMAND} -E create_symlink "${src}" "${dest}" COMMENT "link ${file}" MAIN_DEPENDENCY "${src}" ) # Add a phony target to make sure the files are linked by default. sanitize_target_name("link-" "${file}" NAME) add_custom_target(${NAME} ALL DEPENDS "${dest}") endfunction() make_link(functional/test_runner.py) make_link(util/bitcoin-util-test.py) make_link(util/rpcauth-test.py) make_link(fuzz/test_runner.py) +include(Coverage) include(TestSuite) macro(add_functional_test_check TARGET COMMENT) add_test_custom_target(${TARGET} TEST_COMMAND "${Python_EXECUTABLE}" ./functional/test_runner.py ${ARGN} CUSTOM_TARGET_ARGS COMMENT "${COMMENT}" DEPENDS bitcoind bitcoin-cli ${CMAKE_CURRENT_BINARY_DIR}/functional/test_runner.py USES_TERMINAL VERBATIM ) + + add_custom_target_coverage(${TARGET}) endmacro() add_functional_test_check(check-functional "Run the functional tests" ) add_dependencies(check-all check-functional) add_functional_test_check(check-functional-extended "Run the extended functional tests" --extended ) add_dependencies(check-extended check-functional-extended) set(TEST_SUITE_NAME_UPGRADE_ACTIVATED "Bitcoin ABC functional tests with the next upgrade activated") add_functional_test_check(check-functional-upgrade-activated "Run the functional tests with the upgrade activated" --with-phononactivation -n "${TEST_SUITE_NAME_UPGRADE_ACTIVATED}" ) add_dependencies(check-upgrade-activated check-functional-upgrade-activated) add_functional_test_check(check-functional-upgrade-activated-extended "Run the extended functional tests with the upgrade activated" --extended --with-phononactivation -n "${TEST_SUITE_NAME_UPGRADE_ACTIVATED}" ) add_dependencies(check-upgrade-activated-extended check-functional-upgrade-activated-extended) if(BUILD_BITCOIN_TX) add_test_custom_target(check-bitcoin-util TEST_COMMAND "${Python_EXECUTABLE}" ./util/bitcoin-util-test.py CUSTOM_TARGET_ARGS COMMENT "Test Bitcoin utilities..." DEPENDS bitcoin-tx ${CMAKE_CURRENT_BINARY_DIR}/util/bitcoin-util-test.py ) add_dependencies(check check-bitcoin-util) endif() add_custom_target(check-rpcauth COMMENT "Test Bitcoin RPC authentication..." COMMAND "${Python_EXECUTABLE}" ./util/rpcauth-test.py DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/util/rpcauth-test.py ) add_dependencies(check check-rpcauth) include(PackageHelper) exclude_from_source_package( # Subdirectories "cache/" "lint/" "sanitizer_suppressions/" ) set_property(DIRECTORY "${CMAKE_SOURCE_DIR}" APPEND PROPERTY ADDITIONAL_CLEAN_FILES "${CMAKE_CURRENT_BINARY_DIR}/tmp" "${CMAKE_CURRENT_BINARY_DIR}/cache")