diff --git a/cmake/modules/FindRapidcheck.cmake b/cmake/modules/FindRapidcheck.cmake new file mode 100644 --- /dev/null +++ b/cmake/modules/FindRapidcheck.cmake @@ -0,0 +1,64 @@ +# Copyright (c) 2020 The Bitcoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# .rst: +# FindRapidcheck +# -------------- +# +# Find the Rapidcheck library. +# +# This will define the following variables:: +# +# Rapidcheck_FOUND - True if the Rapidcheck library is found. +# Rapidcheck_INCLUDE_DIRS - List of the header include directories. +# Rapidcheck_LIBRARIES - List of the libraries. +# +# And the following imported targets:: +# +# Rapidcheck::Rapidcheck + +find_path(Rapidcheck_INCLUDE_DIR + NAMES rapidcheck.h + PATH_SUFFIXES rapidcheck +) + +find_library(Rapidcheck_LIBRARY + NAMES rapidcheck + PATH_SUFFIXES rapidcheck +) + +# TODO: extract a version number. +# For now rapidcheck does not provide such a version number, and has no release. +# See https://github.com/emil-e/rapidcheck/issues/235 for reference. + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Rapidcheck + DEFAULT_MSG + Rapidcheck_INCLUDE_DIR + Rapidcheck_LIBRARY +) + +if(Rapidcheck_FOUND) + set(Rapidcheck_INCLUDE_DIRS "${Rapidcheck_INCLUDE_DIR}") + set(Rapidcheck_LIBRARIES "${Rapidcheck_LIBRARY}") + + include(FindPackageMessage) + find_package_message(Rapidcheck + "Found Rapidcheck: ${Rapidcheck_LIBRARIES}" + "[${Rapidcheck_LIBRARIES}][${Rapidcheck_INCLUDE_DIRS}]" + ) + + if(NOT TARGET Rapidcheck::Rapidcheck) + add_library(Rapidcheck::Rapidcheck UNKNOWN IMPORTED) + set_target_properties(Rapidcheck::Rapidcheck PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${Rapidcheck_INCLUDE_DIR}" + IMPORTED_LOCATION "${Rapidcheck_LIBRARY}" + ) + endif() +endif() + +mark_as_advanced( + Rapidcheck_INCLUDE_DIR + Rapidcheck_LIBRARY +) diff --git a/configure.ac b/configure.ac --- a/configure.ac +++ b/configure.ac @@ -124,6 +124,12 @@ [use_gui_tests=$enableval], [use_gui_tests=$use_tests]) +AC_ARG_WITH([rapidcheck], + [AS_HELP_STRING([--with-rapidcheck], + [enable RapidCheck property-based tests (default is yes if librapidcheck is found)])], + [use_rapidcheck=$withval], + [use_rapidcheck=auto]) + AC_ARG_ENABLE(bench, AS_HELP_STRING([--disable-bench],[do not compile benchmarks (default is to compile)]), [use_bench=$enableval], @@ -1047,6 +1053,22 @@ ]) CXXFLAGS="${save_CXXFLAGS}" +dnl RapidCheck property-based testing + +enable_property_tests=no +if test "x$use_rapidcheck" = xauto; then + AC_CHECK_HEADERS([rapidcheck.h], [enable_property_tests=yes]) +elif test "x$use_rapidcheck" != xno; then + enable_property_tests=yes +fi + +RAPIDCHECK_LIBS= +if test "x$enable_property_tests" = xyes; then + RAPIDCHECK_LIBS=-lrapidcheck +fi +AC_SUBST(RAPIDCHECK_LIBS) +AM_CONDITIONAL([ENABLE_PROPERTY_TESTS], [test x$enable_property_tests = xyes]) + dnl univalue check need_bundled_univalue=yes @@ -1403,6 +1425,7 @@ echo " with zmq = $use_zmq" echo " with test = $use_tests" if test x$use_tests != xno; then + echo " with prop = $enable_property_tests" echo " with fuzz = $enable_fuzz" fi echo " with bench = $use_bench" diff --git a/depends/Makefile b/depends/Makefile --- a/depends/Makefile +++ b/depends/Makefile @@ -4,6 +4,7 @@ BASE_CACHE ?= $(BASEDIR)/built SDK_PATH ?= $(BASEDIR)/SDKs NO_QT ?= +RAPIDCHECK ?= NO_WALLET ?= NO_UPNP ?= FALLBACK_DOWNLOAD_PATH ?= https://download.bitcoinabc.org/depends-sources @@ -94,6 +95,8 @@ wallet_packages_$(NO_WALLET) = $(wallet_packages) upnp_packages_$(NO_UPNP) = $(upnp_packages) +rapidcheck_packages_$(RAPIDCHECK) = $(rapidcheck_packages) + packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) @@ -101,6 +104,10 @@ native_packages += $(qt_native_packages) endif +ifeq ($(rapidcheck_packages_),) +packages += $(rapidcheck_packages) +endif + all_packages = $(packages) $(native_packages) meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk diff --git a/depends/README.md b/depends/README.md --- a/depends/README.md +++ b/depends/README.md @@ -39,6 +39,7 @@ NO_WALLET: Don't download/build/cache libs needed to enable the wallet NO_UPNP: Don't download/build/cache packages needed for enabling upnp DEBUG: disable some optimizations and enable more runtime checking + RAPIDCHECK: build rapidcheck (experimental) HOST_ID_SALT: Optional salt to use when generating host package ids BUILD_ID_SALT: Optional salt to use when generating build package ids JOBS: Number of jobs to use for each package build diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -6,6 +6,8 @@ qt_linux_packages:=qt expat dbus libxcb xcb_proto libXau xproto freetype fontconfig libX11 xextproto libXext xtrans +rapidcheck_packages = rapidcheck + qt_darwin_packages=qt qt_mingw32_packages=qt diff --git a/depends/packages/rapidcheck.mk b/depends/packages/rapidcheck.mk new file mode 100644 --- /dev/null +++ b/depends/packages/rapidcheck.mk @@ -0,0 +1,18 @@ +package=rapidcheck +$(package)_version=10fc0cb +$(package)_download_path=https://github.com/MarcoFalke/rapidcheck/archive +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=9640926223c00af45bce4c7df8b756b5458a89b2ba74cfe3e404467f13ce26df + +define $(package)_config_cmds + cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true . +endef + +define $(package)_build_cmds + $(MAKE) -j$(JOBS) && \ + mkdir -p $($(package)_staging_dir)$(host_prefix)/include && \ + cp -a include/* $($(package)_staging_dir)$(host_prefix)/include/ && \ + cp -a extras/boost_test/include/rapidcheck/* $($(package)_staging_dir)$(host_prefix)/include/rapidcheck/ && \ + mkdir -p $($(package)_staging_dir)$(host_prefix)/lib && \ + cp -a librapidcheck.a $($(package)_staging_dir)$(host_prefix)/lib/ +endef diff --git a/doc/rapidcheck.md b/doc/rapidcheck.md new file mode 100644 --- /dev/null +++ b/doc/rapidcheck.md @@ -0,0 +1,69 @@ +# RapidCheck property-based testing for Bitcoin ABC + +## Concept + +Property-based testing is experimentally being added to Bitcoin ABC with +[RapidCheck](https://github.com/emil-e/rapidcheck), a C++ framework for +property-based testing inspired by the Haskell library +[QuickCheck](https://hackage.haskell.org/package/QuickCheck). + +RapidCheck performs random testing of program properties. A specification of the +program is given in the form of properties which functions should satisfy, and +RapidCheck tests that the properties hold in a large number of randomly +generated cases. + +If an exception is found, RapidCheck tries to find the smallest case, for some +definition of smallest, for which the property is still false and displays it as +a counter-example. For example, if the input is an integer, RapidCheck tries to +find the smallest integer for which the property is false. + +## Setup + +The following instructions have been tested with Linux Debian and macOS. + +1. Clone the RapidCheck source code and cd into the repository. + + ```shell + git clone https://github.com/emil-e/rapidcheck.git + cd rapidcheck + ``` + +2. Build RapidCheck. + + ```shell + mkdir build + cd build + cmake -GNinja .. -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DRC_ENABLE_BOOST_TEST=ON + ninja + sudo ninja install/strip + ``` + +3. Configure Bitcoin ABC with RapidCheck. + + `cd` to the directory of your local Bitcoin ABC repository, create a build + directory and run cmake: + + ```shell + mkdir build + cd build + cmake -GNinja .. -DENABLE_PROPERTY_BASED_TESTS=ON + ``` + + In the output you should see something similar to: + + ```shell + [...] + -- Found Rapidcheck: /usr/local/include + -- Found Rapidcheck: /usr/local/lib/librapidcheck.a + [...] + ``` + +4. Build Bitcoin ABC with RapidCheck. + + Now you can run `ninja check` to build and run the unit tests, including the + property-based tests. You can also build and run a single test by using + `ninja check-bitcoin-`. + + Example: `ninja check-bitcoin-key_properties` + +That's it! You are now running property-based tests in Bitcoin ABC. diff --git a/src/Makefile.test.include b/src/Makefile.test.include --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -34,6 +34,7 @@ TEST_BINARY=test/test_bitcoin$(EXEEXT) JSON_TEST_FILES = \ + test/data/script_tests.json \ test/data/base58_encode_decode.json \ test/data/blockfilters.json \ test/data/key_io_valid.json \ @@ -167,6 +168,15 @@ test/work_comparator_tests.cpp \ rpc/test/server_tests.cpp +if ENABLE_PROPERTY_TESTS +BITCOIN_TESTS += \ + test/key_properties.cpp + +BITCOIN_TEST_SUITE += \ + test/gen/crypto_gen.cpp \ + test/gen/crypto_gen.h +endif + if ENABLE_WALLET BITCOIN_TESTS += \ wallet/test/accounting_tests.cpp \ @@ -196,7 +206,7 @@ $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_test_bitcoin_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) +test_test_bitcoin_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(RAPIDCHECK_LIBS) test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static if ENABLE_ZMQ diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -2,6 +2,8 @@ project(bitcoin-test) +option(ENABLE_PROPERTY_BASED_TESTS "Enable property based tests" OFF) + # Process json files. file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/data") @@ -60,6 +62,17 @@ ) endif() +if(ENABLE_PROPERTY_BASED_TESTS) + find_package(Rapidcheck REQUIRED) + + set(BITCOIN_PROPERTY_BASED_TEST_FIXTURE + gen/crypto_gen.cpp + ) + set(BITCOIN_PROPERTY_BASED_TESTS + key_properties.cpp + ) +endif() + add_boost_unit_tests_to_suite(bitcoin test_bitcoin jsonutil.cpp scriptflags.cpp @@ -73,6 +86,9 @@ # Wallet test fixture ${BITCOIN_WALLET_TEST_FIXTURE} + # Properties based test fixture + ${BITCOIN_PROPERTY_BASED_TEST_FIXTURE} + TESTS activation_tests.cpp addrman_tests.cpp @@ -179,8 +195,15 @@ # Wallet tests ${BITCOIN_WALLET_TESTS} + + # Properties based tests + ${BITCOIN_PROPERTY_BASED_TESTS} ) +if(ENABLE_PROPERTY_BASED_TESTS) + target_link_libraries(test_bitcoin Rapidcheck::Rapidcheck) +endif() + function(add_boost_test_runners_with_upgrade_activated SUITE EXECUTABLE) set(SUITE_UPGRADE_ACTIVATED "${SUITE}-upgrade-activated") get_target_from_suite(${SUITE_UPGRADE_ACTIVATED} TARGET_UPGRADE_ACTIVATED) diff --git a/src/test/gen/crypto_gen.h b/src/test/gen/crypto_gen.h new file mode 100644 --- /dev/null +++ b/src/test/gen/crypto_gen.h @@ -0,0 +1,51 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#ifndef BITCOIN_TEST_GEN_CRYPTO_GEN_H +#define BITCOIN_TEST_GEN_CRYPTO_GEN_H + +#include +#include +#include + +#include +#include +#include +#include + +/** Generates 1 to 15 keys for OP_CHECKMULTISIG */ +rc::Gen> MultisigKeys(); + +namespace rc { +/** Generator for a new CKey */ +template <> struct Arbitrary { + static Gen arbitrary() { + return rc::gen::map([](int x) { + CKey key; + key.MakeNewKey(true); + return key; + }); + }; +}; + +/** Generator for a CPrivKey */ +template <> struct Arbitrary { + static Gen arbitrary() { + return gen::map(gen::arbitrary(), + [](const CKey &key) { return key.GetPrivKey(); }); + }; +}; + +/** Generator for a new CPubKey */ +template <> struct Arbitrary { + static Gen arbitrary() { + return gen::map(gen::arbitrary(), + [](const CKey &key) { return key.GetPubKey(); }); + }; +}; +/** Generates a arbitrary uint256 */ +template <> struct Arbitrary { + static Gen arbitrary() { return rc::gen::just(GetRandHash()); }; +}; +} // namespace rc +#endif // BITCOIN_TEST_GEN_CRYPTO_GEN_H diff --git a/src/test/gen/crypto_gen.cpp b/src/test/gen/crypto_gen.cpp new file mode 100644 --- /dev/null +++ b/src/test/gen/crypto_gen.cpp @@ -0,0 +1,19 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include + +#include + +#include +#include +#include +#include + +/** Generates 1 to 20 keys for OP_CHECKMULTISIG */ +rc::Gen> MultisigKeys() { + return rc::gen::suchThat(rc::gen::arbitrary>(), + [](const std::vector &keys) { + return keys.size() >= 1 && keys.size() <= 15; + }); +}; diff --git a/src/test/key_properties.cpp b/src/test/key_properties.cpp new file mode 100644 --- /dev/null +++ b/src/test/key_properties.cpp @@ -0,0 +1,60 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include + +#include +#include + +#include + +#include +#include +#include + +#include +#include + +BOOST_FIXTURE_TEST_SUITE(key_properties, BasicTestingSetup) + +/** Check CKey uniqueness */ +RC_BOOST_PROP(key_uniqueness, (const CKey &key1, const CKey &key2)) { + RC_ASSERT(!(key1 == key2)); +} + +/** Verify that a private key generates the correct public key */ +RC_BOOST_PROP(key_generates_correct_pubkey, (const CKey &key)) { + CPubKey pubKey = key.GetPubKey(); + RC_ASSERT(key.VerifyPubKey(pubKey)); +} + +/** Create a CKey using the 'Set' function must give us the same key */ +RC_BOOST_PROP(key_set_symmetry, (const CKey &key)) { + CKey key1; + key1.Set(key.begin(), key.end(), key.IsCompressed()); + RC_ASSERT(key1 == key); +} + +/** + * Create a CKey, sign a piece of data with ECDSA algorithm, then verify it + * with the public key. + */ +RC_BOOST_PROP(key_sign_symmetry_ecdsa, (const CKey &key, const uint256 &hash)) { + std::vector vchSig; + key.SignECDSA(hash, vchSig, 0); + const CPubKey &pubKey = key.GetPubKey(); + RC_ASSERT(pubKey.VerifyECDSA(hash, vchSig)); +} + +/** + * Create a CKey, sign a piece of data with Schnorr algorithm, then verify it + * with the public key. + */ +RC_BOOST_PROP(key_sign_symmetry_schnorr, + (const CKey &key, const uint256 &hash)) { + std::vector vchSig; + key.SignSchnorr(hash, vchSig, 0); + const CPubKey &pubKey = key.GetPubKey(); + RC_ASSERT(pubKey.VerifySchnorr(hash, vchSig)); +} +BOOST_AUTO_TEST_SUITE_END()