diff --git a/CMakeLists.txt b/CMakeLists.txt index fda3950b7..dbaf31569 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,81 +1,81 @@ # Copyright (c) 2017 The Bitcoin developers cmake_minimum_required(VERSION 3.12) project(bitcoin-abc VERSION 0.20.6 DESCRIPTION "Bitcoin ABC is a full node implementation of the Bitcoin Cash protocol." HOMEPAGE_URL "https://www.bitcoinabc.org" ) # Package information set(PACKAGE_NAME "Bitcoin ABC") # Copyright set(COPYRIGHT_YEAR 2019) set(COPYRIGHT_HOLDERS "The %s developers") set(COPYRIGHT_HOLDERS_SUBSTITUTION Bitcoin) string(REPLACE "%s" ${COPYRIGHT_HOLDERS_SUBSTITUTION} COPYRIGHT_HOLDERS_FINAL ${COPYRIGHT_HOLDERS}) # Add path for custom modules set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules ) # Make contrib script accessible. set(CONTRIB_PATH ${CMAKE_CURRENT_SOURCE_DIR}/contrib) # If ccache is available, then use it. find_program(CCACHE ccache) if(CCACHE) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE}) set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE}) endif(CCACHE) # Default to RelWithDebInfo configuration if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Select the configuration for the build" FORCE) endif() # Find the python interpreter. This is required for several targets. -find_package(PythonInterp 3.4 REQUIRED) +find_package(PythonInterp 3.5 REQUIRED) # Add the magic targets `check-*` add_custom_target(check-all) add_custom_target(check) add_custom_target(check-symbols) add_custom_target(check-security) include(PackageHelper) exclude_git_ignored_files_from_source_package() # Ignore hidden files and directories (starting with a '.') set_property(GLOBAL APPEND PROPERTY SOURCE_PACKAGE_IGNORE_FILES "/\\\\.") # If the build is out-of-tree, then the build directory can be ignored. if(NOT CMAKE_BINARY_DIR STREQUAL CMAKE_SOURCE_DIR) set_property(GLOBAL APPEND PROPERTY SOURCE_PACKAGE_IGNORE_FILES "${CMAKE_BINARY_DIR}/" ) endif() exclude_from_source_package( # Subdirectories "arcanist/" "depends/" # Files "[^.]+[.]md$" "Dockerfile-doxygen" ) add_subdirectory(src) add_subdirectory(test) add_subdirectory(contrib) add_subdirectory(doc) include(PackageOptions) get_property(CPACK_SOURCE_IGNORE_FILES GLOBAL PROPERTY SOURCE_PACKAGE_IGNORE_FILES) include(CPack) diff --git a/configure.ac b/configure.ac index 26ec69247..1397ce3af 100644 --- a/configure.ac +++ b/configure.ac @@ -1,1351 +1,1351 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N) AC_PREREQ([2.60]) define(_CLIENT_VERSION_MAJOR, 0) define(_CLIENT_VERSION_MINOR, 20) define(_CLIENT_VERSION_REVISION, 6) define(_CLIENT_VERSION_IS_RELEASE, true) define(_COPYRIGHT_YEAR, 2019) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[Bitcoin]]) AC_INIT([Bitcoin ABC],[_CLIENT_VERSION_MAJOR._CLIENT_VERSION_MINOR._CLIENT_VERSION_REVISION],[https://github.com/Bitcoin-ABC/bitcoin-abc/issues],[bitcoin-abc],[https://bitcoinabc.org/]) AC_CONFIG_SRCDIR([src/validation.cpp]) AC_CONFIG_HEADERS([src/config/bitcoin-config.h]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) BITCOIN_DAEMON_NAME=bitcoind BITCOIN_GUI_NAME=bitcoin-qt BITCOIN_CLI_NAME=bitcoin-cli BITCOIN_TX_NAME=bitcoin-tx BITCOIN_SEEDER_NAME=bitcoin-seeder dnl Unless the user specified ARFLAGS, force it to be cr AC_ARG_VAR(ARFLAGS, [Flags for the archiver, defaults to if not set]) if test "x${ARFLAGS+set}" != "xset"; then ARFLAGS="cr" fi AC_CANONICAL_HOST AH_TOP([#ifndef BITCOIN_BITCOIN_CONFIG_H]) AH_TOP([#define BITCOIN_BITCOIN_CONFIG_H]) AH_BOTTOM([#endif // BITCOIN_BITCOIN_CONFIG_H]) dnl faketime breaks configure and is only needed for make. Disable it here. unset FAKETIME dnl Automake init set-up and checks AM_INIT_AUTOMAKE([no-define subdir-objects foreign]) dnl faketime messes with timestamps and causes configure to be re-run. dnl --disable-maintainer-mode can be used to bypass this. AM_MAINTAINER_MODE([enable]) dnl make the compilation flags quiet unless V=1 is used m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) dnl Compiler checks (here before libtool). if test "x${CXXFLAGS+set}" = "xset"; then CXXFLAGS_overridden=yes else CXXFLAGS_overridden=no fi AC_PROG_CXX dnl By default, libtool for mingw refuses to link static libs into a dll for dnl fear of mixing pic/non-pic objects, and import/export complications. Since dnl we have those under control, re-enable that functionality. case $host in *mingw*) lt_cv_deplibs_check_method="pass_all" ;; esac dnl Require C++14 compiler (no GNU extensions) AX_CXX_COMPILE_STDCXX([14], [noext], [mandatory], [nodefault]) dnl Check if -latomic is required for CHECK_ATOMIC dnl Unless the user specified OBJCXX, force it to be the same as CXX. This ensures dnl that we get the same -std flags for both. m4_ifdef([AC_PROG_OBJCXX],[ if test "x${OBJCXX+set}" = "x"; then OBJCXX="${CXX}" fi AC_PROG_OBJCXX ]) dnl Libtool init checks. LT_INIT([pic-only]) dnl Check/return PATH for base programs. AC_PATH_TOOL(AR, ar) AC_PATH_TOOL(RANLIB, ranlib) AC_PATH_TOOL(STRIP, strip) AC_PATH_TOOL(GCOV, gcov) AC_PATH_PROG(LCOV, lcov) -dnl Python 3.x is supported from 3.4 on (see https://github.com/bitcoin/bitcoin/issues/7893) -AC_PATH_PROGS([PYTHON], [python3.7 python3.6 python3.5 python3.4 python3 python]) +dnl Python 3.x is supported from 3.5 +AC_PATH_PROGS([PYTHON], [python3.8 python3.7 python3.6 python3.5 python3 python]) AC_PATH_PROG(GENHTML, genhtml) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) AC_PATH_PROG(XGETTEXT,xgettext) AC_PATH_PROG(HEXDUMP,hexdump) AC_PATH_TOOL(READELF, readelf) AC_PATH_TOOL(CPPFILT, c++filt) AC_PATH_TOOL(OBJCOPY, objcopy) AC_ARG_VAR(PYTHONPATH, Augments the default search path for python module files) # Enable wallet AC_ARG_ENABLE([wallet], [AS_HELP_STRING([--disable-wallet], [disable wallet (enabled by default)])], [enable_wallet=$enableval], [enable_wallet=yes]) AC_ARG_WITH([miniupnpc], [AS_HELP_STRING([--with-miniupnpc], [enable UPNP (default is yes if libminiupnpc is found)])], [use_upnp=$withval], [use_upnp=auto]) AC_ARG_ENABLE([upnp-default], [AS_HELP_STRING([--enable-upnp-default], [if UPNP is enabled, turn it on at startup (default is no)])], [use_upnp_default=$enableval], [use_upnp_default=no]) AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[do not compile tests (default is to compile)]), [use_tests=$enableval], [use_tests=yes]) AC_ARG_ENABLE(gui-tests, AS_HELP_STRING([--disable-gui-tests],[do not compile GUI tests (default is to compile if GUI and tests enabled)]), [use_gui_tests=$enableval], [use_gui_tests=$use_tests]) AC_ARG_ENABLE(bench, AS_HELP_STRING([--disable-bench],[do not compile benchmarks (default is to compile)]), [use_bench=$enableval], [use_bench=yes]) AC_ARG_ENABLE([extended-functional-tests], AS_HELP_STRING([--enable-extended-functional-tests],[enable expensive functional tests when using lcov (default no)]), [use_extended_functional_tests=$enableval], [use_extended_functional_tests=no]) AC_ARG_WITH([qrencode], [AS_HELP_STRING([--with-qrencode], [enable QR code support (default is yes if qt is enabled and libqrencode is found)])], [use_qr=$withval], [use_qr=auto]) AC_ARG_ENABLE([hardening], [AS_HELP_STRING([--disable-hardening], [do not attempt to harden the resulting executables (default is to harden)])], [use_hardening=$enableval], [use_hardening=yes]) AC_ARG_ENABLE([reduce-exports], [AS_HELP_STRING([--enable-reduce-exports], [attempt to reduce exported symbols in the resulting executables (default is no)])], [use_reduce_exports=$enableval], [use_reduce_exports=no]) AC_ARG_ENABLE([ccache], [AS_HELP_STRING([--disable-ccache], [do not use ccache for building (default is to use if found)])], [use_ccache=$enableval], [use_ccache=auto]) AC_ARG_ENABLE([lcov], [AS_HELP_STRING([--enable-lcov], [enable lcov testing (default is no)])], [use_lcov=$enableval], [use_lcov=no]) AC_ARG_ENABLE([lcov-branch-coverage], [AS_HELP_STRING([--enable-lcov-branch-coverage], [enable lcov testing branch coverage (default is no)])], [use_lcov_branch=yes], [use_lcov_branch=no]) AC_ARG_ENABLE([glibc-back-compat], [AS_HELP_STRING([--enable-glibc-back-compat], [enable backwards compatibility with glibc])], [use_glibc_compat=$enableval], [use_glibc_compat=no]) AC_ARG_ENABLE([asm], [AS_HELP_STRING([--enable-asm], [Enable assembly routines (default is yes)])], [use_asm=$enableval], [use_asm=yes]) if test "x$use_asm" = xyes; then AC_DEFINE(USE_ASM, 1, [Define this symbol to build in assembly routines]) fi AC_ARG_WITH([system-univalue], [AS_HELP_STRING([--with-system-univalue], [Build with system UniValue (default is no)])], [system_univalue=$withval], [system_univalue=no] ) AC_ARG_ENABLE([zmq], [AS_HELP_STRING([--disable-zmq], [disable ZMQ notifications])], [use_zmq=$enableval], [use_zmq=yes]) AC_ARG_WITH([protoc-bindir],[AS_HELP_STRING([--with-protoc-bindir=BIN_DIR],[specify protoc bin path])], [protoc_bin_path=$withval], []) AC_ARG_ENABLE(man, [AS_HELP_STRING([--disable-man], [do not install man pages (default is to install)])],, enable_man=yes) AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no) # Enable debug AC_ARG_ENABLE([debug], [AS_HELP_STRING([--enable-debug], [use debug compiler flags and macros (default is no)])], [enable_debug=$enableval], [enable_debug=no]) # Enable different -fsanitize options AC_ARG_WITH([sanitizers], [AS_HELP_STRING([--with-sanitizers], [comma separated list of extra sanitizers to build with (default is none enabled)])], [use_sanitizers=$withval]) # Turn warnings into errors AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], [Treat certain compiler warnings as errors (default is no)])], [enable_werror=$enableval], [enable_werror=no]) AC_LANG_PUSH([C++]) AX_CHECK_COMPILE_FLAG([-Werror],[CXXFLAG_WERROR="-Werror"],[CXXFLAG_WERROR=""]) if test "x$enable_debug" = xyes; then # Clear default -g -O2 flags if test "x$CXXFLAGS_overridden" = xno; then CXXFLAGS="" fi # Prefer -Og, fall back to -O0 if that is unavailable. AX_CHECK_COMPILE_FLAG( [-Og], [[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -Og"]], [AX_CHECK_COMPILE_FLAG([-O0],[[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -O0"]],,[[$CXXFLAG_WERROR]])], [[$CXXFLAG_WERROR]]) # Prefer -g3, fall back to -g if that is unavailable. AX_CHECK_COMPILE_FLAG( [-g3], [[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g3"]], [AX_CHECK_COMPILE_FLAG([-g],[[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g"]],,[[$CXXFLAG_WERROR]])], [[$CXXFLAG_WERROR]]) AX_CHECK_PREPROC_FLAG([-DDEBUG],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKORDER],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKORDER"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-ftrapv],[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"],,[[$CXXFLAG_WERROR]]) fi if test x$use_sanitizers != x; then # First check if the compiler accepts flags. If an incompatible pair like # -fsanitize=address,thread is used here, this check will fail. This will also # fail if a bad argument is passed, e.g. -fsanitize=undfeined AX_CHECK_COMPILE_FLAG( [[-fsanitize=$use_sanitizers]], [[SANITIZER_CXXFLAGS=-fsanitize=$use_sanitizers]], [AC_MSG_ERROR([compiler did not accept requested flags])]) # Some compilers (e.g. GCC) require additional libraries like libasan, # libtsan, libubsan, etc. Make sure linking still works with the sanitize # flag. This is a separate check so we can give a better error message when # the sanitize flags are supported by the compiler but the actual sanitizer # libs are missing. AX_CHECK_LINK_FLAG( [[-fsanitize=$use_sanitizers]], [[SANITIZER_LDFLAGS=-fsanitize=$use_sanitizers]], [AC_MSG_ERROR([linker did not accept requested flags, you are missing required libraries])]) fi ERROR_CXXFLAGS= if test "x$enable_werror" = "xyes"; then if test "x$CXXFLAG_WERROR" = "x"; then AC_MSG_ERROR("enable-werror set but -Werror is not usable") fi AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=thread-safety-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety-analysis"],,[[$CXXFLAG_WERROR]]) fi if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wextra],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wvla],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat-security],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat-security"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wthread-safety-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wshadow],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wmissing-braces],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wmissing-braces"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wredundant-move],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-move"],,[[$CXXFLAG_WERROR]]) ## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all ## unknown options if any other warning is produced. Test the -Wfoo case, and ## set the -Wno-foo case if it works. AX_CHECK_COMPILE_FLAG([-Wunused-parameter],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-implicit-fallthrough"],,[[$CXXFLAG_WERROR]]) fi # Check for optional instruction set support. Enabling these does _not_ imply that all code will # be compiled with them, rather that specific objects/libs may use them after checking for runtime # compatibility. AX_CHECK_COMPILE_FLAG([-msse4.2],[[SSE42_CXXFLAGS="-msse4.2"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-msse4.1],[[SSE41_CXXFLAGS="-msse4.1"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-mavx -mavx2],[[AVX2_CXXFLAGS="-mavx -mavx2"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-msse4 -msha],[[SHANI_CXXFLAGS="-msse4 -msha"]],,[[$CXXFLAG_WERROR]]) TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS" AC_MSG_CHECKING(for assembler crc32 support) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #if defined(_MSC_VER) #include #elif defined(__GNUC__) && defined(__SSE4_2__) #include #endif ]],[[ uint64_t l = 0; l = _mm_crc32_u8(l, 0); l = _mm_crc32_u32(l, 0); l = _mm_crc32_u64(l, 0); return l; ]])], [ AC_MSG_RESULT(yes); enable_hwcrc32=yes], [ AC_MSG_RESULT(no)] ) CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $SSE41_CXXFLAGS" AC_MSG_CHECKING(for SSE4.1 intrinsics) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #include ]],[[ __m128i l = _mm_set1_epi32(0); return _mm_extract_epi32(l, 3); ]])], [ AC_MSG_RESULT(yes); enable_sse41=yes; AC_DEFINE(ENABLE_SSE41, 1, [Define this symbol to build code that uses SSE4.1 intrinsics]) ], [ AC_MSG_RESULT(no)] ) CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $AVX2_CXXFLAGS" AC_MSG_CHECKING(for AVX2 intrinsics) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #include ]],[[ __m256i l = _mm256_set1_epi32(0); return _mm256_extract_epi32(l, 7); ]])], [ AC_MSG_RESULT(yes); enable_avx2=yes; AC_DEFINE(ENABLE_AVX2, 1, [Define this symbol to build code that uses AVX2 intrinsics]) ], [ AC_MSG_RESULT(no)] ) CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $SHANI_CXXFLAGS" AC_MSG_CHECKING(for SHA-NI intrinsics) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #include ]],[[ __m128i i = _mm_set1_epi32(0); __m128i j = _mm_set1_epi32(1); __m128i k = _mm_set1_epi32(2); return _mm_extract_epi32(_mm_sha256rnds2_epu32(i, i, k), 0); ]])], [ AC_MSG_RESULT(yes); enable_shani=yes; AC_DEFINE(ENABLE_SHANI, 1, [Define this symbol to build code that uses SHA-NI intrinsics]) ], [ AC_MSG_RESULT(no)] ) CXXFLAGS="$TEMP_CXXFLAGS" CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], [build bitcoin-cli bitcoin-tx (default=yes)])], [build_bitcoin_utils=$withval], [build_bitcoin_utils=yes]) AC_ARG_WITH([libs], [AS_HELP_STRING([--with-libs], [build libraries (default=yes)])], [build_bitcoin_libs=$withval], [build_bitcoin_libs=yes]) AC_ARG_WITH([daemon], [AS_HELP_STRING([--with-daemon], [build bitcoind daemon (default=yes)])], [build_bitcoind=$withval], [build_bitcoind=yes]) AC_ARG_WITH([seeder], [AS_HELP_STRING([--with-seeder], [build seeder (default=yes)])], [build_bitcoin_seeder=$withval], [build_bitcoin_seeder=yes]) use_pkgconfig=yes case $host in *mingw*) #pkgconfig does more harm than good with MinGW use_pkgconfig=no TARGET_OS=windows AC_CHECK_LIB([mingwthrd], [main],, AC_MSG_ERROR(libmingwthrd missing)) AC_CHECK_LIB([kernel32], [main],, AC_MSG_ERROR(libkernel32 missing)) AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) AC_CHECK_LIB([gdi32], [main],, AC_MSG_ERROR(libgdi32 missing)) AC_CHECK_LIB([comdlg32], [main],, AC_MSG_ERROR(libcomdlg32 missing)) AC_CHECK_LIB([winspool], [main],, AC_MSG_ERROR(libwinspool missing)) AC_CHECK_LIB([winmm], [main],, AC_MSG_ERROR(libwinmm missing)) AC_CHECK_LIB([shell32], [main],, AC_MSG_ERROR(libshell32 missing)) AC_CHECK_LIB([comctl32], [main],, AC_MSG_ERROR(libcomctl32 missing)) AC_CHECK_LIB([ole32], [main],, AC_MSG_ERROR(libole32 missing)) AC_CHECK_LIB([oleaut32], [main],, AC_MSG_ERROR(liboleaut32 missing)) AC_CHECK_LIB([uuid], [main],, AC_MSG_ERROR(libuuid missing)) AC_CHECK_LIB([rpcrt4], [main],, AC_MSG_ERROR(librpcrt4 missing)) AC_CHECK_LIB([advapi32], [main],, AC_MSG_ERROR(libadvapi32 missing)) AC_CHECK_LIB([ws2_32], [main],, AC_MSG_ERROR(libws2_32 missing)) AC_CHECK_LIB([mswsock], [main],, AC_MSG_ERROR(libmswsock missing)) AC_CHECK_LIB([shlwapi], [main],, AC_MSG_ERROR(libshlwapi missing)) AC_CHECK_LIB([iphlpapi], [main],, AC_MSG_ERROR(libiphlpapi missing)) AC_CHECK_LIB([crypt32], [main],, AC_MSG_ERROR(libcrypt32 missing)) # -static is interpreted by libtool, where it has a different meaning. # In libtool-speak, it's -all-static. AX_CHECK_LINK_FLAG([[-static]],[LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static"]) AC_PATH_PROG([MAKENSIS], [makensis], none) if test x$MAKENSIS = xnone; then AC_MSG_WARN("makensis not found. Cannot create installer.") fi AC_PATH_TOOL(WINDRES, windres, none) if test x$WINDRES = xnone; then AC_MSG_ERROR("windres not found") fi CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB" LEVELDB_TARGET_FLAGS="-DOS_WINDOWS" if test "x$CXXFLAGS_overridden" = "xno"; then CXXFLAGS="$CXXFLAGS -w" fi case $host in i?86-*) WINDOWS_BITS=32 ;; x86_64-*) WINDOWS_BITS=64 ;; *) AC_MSG_ERROR("Could not determine win32/win64 for installer") ;; esac AC_SUBST(WINDOWS_BITS) dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override dnl its command here, with the predeps/postdeps removed, and -static inserted. Postdeps are dnl also overridden to prevent their insertion later. dnl This should only affect dll's. archive_cmds_CXX="\$CC -shared \$libobjs \$deplibs \$compiler_flags -static -o \$output_objdir/\$soname \${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker \$lib" postdeps_CXX= ;; *darwin*) TARGET_OS=darwin LEVELDB_TARGET_FLAGS="-DOS_MACOSX" if test x$cross_compiling != xyes; then BUILD_OS=darwin AC_CHECK_PROG([PORT],port, port) if test x$PORT = xport; then dnl add default macports paths CPPFLAGS="$CPPFLAGS -isystem /opt/local/include" LIBS="$LIBS -L/opt/local/lib" if test -d /opt/local/include/db48; then CPPFLAGS="$CPPFLAGS -I/opt/local/include/db48" LIBS="$LIBS -L/opt/local/lib/db48" fi fi AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) AC_CHECK_PROG([BREW],brew, brew) if test x$BREW = xbrew; then dnl These Homebrew packages may be keg-only, meaning that they won't be found dnl in expected paths because they may conflict with system files. Ask dnl Homebrew where each one is located, then adjust paths accordingly. dnl It's safe to add these paths even if the functionality is disabled by dnl the user (--without-wallet or --without-gui for example). openssl_prefix=`$BREW --prefix openssl 2>/dev/null` bdb_prefix=`$BREW --prefix berkeley-db 2>/dev/null` qt5_prefix=`$BREW --prefix qt5 2>/dev/null` if test x$openssl_prefix != x; then PKG_CONFIG_PATH="$openssl_prefix/lib/pkgconfig:$PKG_CONFIG_PATH" export PKG_CONFIG_PATH fi if test x$bdb_prefix != x; then CPPFLAGS="$CPPFLAGS -I$bdb_prefix/include" LIBS="$LIBS -L$bdb_prefix/lib" fi if test x$qt5_prefix != x; then PKG_CONFIG_PATH="$qt5_prefix/lib/pkgconfig:$PKG_CONFIG_PATH" export PKG_CONFIG_PATH fi fi else case $build_os in *darwin*) BUILD_OS=darwin ;; *) AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool) AC_PATH_TOOL([OTOOL], [otool], otool) AC_PATH_PROGS([GENISOIMAGE], [genisoimage mkisofs],genisoimage) AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert) AC_PATH_PROGS([TIFFCP], [tiffcp],tiffcp) dnl libtool will try to strip the static lib, which is a problem for dnl cross-builds because strip attempts to call a hard-coded ld, dnl which may not exist in the path. Stripping the .a is not dnl necessary, so just disable it. old_striplib= ;; esac fi AX_CHECK_LINK_FLAG([[-Wl,-headerpad_max_install_names]], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"]) CPPFLAGS="$CPPFLAGS -DMAC_OSX" OBJCXXFLAGS="$CXXFLAGS" ;; *android*) dnl make sure android stays above linux for hosts like *linux-android* LEVELDB_TARGET_FLAGS="-DOS_ANDROID" ;; *linux*) TARGET_OS=linux LEVELDB_TARGET_FLAGS="-DOS_LINUX" ;; *kfreebsd*) LEVELDB_TARGET_FLAGS="-DOS_KFREEBSD" ;; *freebsd*) LEVELDB_TARGET_FLAGS="-DOS_FREEBSD" ;; *openbsd*) LEVELDB_TARGET_FLAGS="-DOS_OPENBSD" ;; *netbsd*) LEVELDB_TARGET_FLAGS="-DOS_NETBSD" ;; *dragonfly*) LEVELDB_TARGET_FLAGS="-DOS_DRAGONFLYBSD" ;; *solaris*) LEVELDB_TARGET_FLAGS="-DOS_SOLARIS" ;; *hpux*) LEVELDB_TARGET_FLAGS="-DOS_HPUX" ;; *) AC_MSG_ERROR(Cannot build leveldb for $host. Please file a bug report.) ;; esac if test x$use_pkgconfig = xyes; then m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR(PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh.)]) m4_ifdef([PKG_PROG_PKG_CONFIG], [ PKG_PROG_PKG_CONFIG if test x"$PKG_CONFIG" = "x"; then AC_MSG_ERROR(pkg-config not found.) fi ]) fi if test x$use_extended_functional_tests != xno; then AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended) fi if test x$use_lcov = xyes; then if test x$LCOV = x; then AC_MSG_ERROR("lcov testing requested but lcov not found") fi if test x$GCOV = x; then AC_MSG_ERROR("lcov testing requested but gcov not found") fi if test x$PYTHON = x; then AC_MSG_ERROR("lcov testing requested but python not found") fi if test x$GENHTML = x; then AC_MSG_ERROR("lcov testing requested but genhtml not found") fi LCOV="$LCOV --gcov-tool=$GCOV" AX_CHECK_LINK_FLAG([[--coverage]], [LDFLAGS="$LDFLAGS --coverage"], [AC_MSG_ERROR("lcov testing requested but --coverage linker flag does not work")]) AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage"], [AC_MSG_ERROR("lcov testing requested but --coverage flag does not work")]) AC_DEFINE(USE_COVERAGE, 1, [Define this symbol if coverage is enabled]) CXXFLAGS="$CXXFLAGS -Og" fi if test x$use_lcov_branch != xno; then AC_SUBST(LCOV_OPTS, "$LCOV_OPTS --rc lcov_branch_coverage=1") fi dnl Check for endianness AC_C_BIGENDIAN dnl Check for pthread compile/link requirements AX_PTHREAD # Check for Large File Supprot before AC_SYS_LARGEFILE gets called as it will # change the output. AC_MSG_CHECKING(for if LFS is enabled) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ static_assert(sizeof(off_t) == 8, ""); ]])], [ AC_MSG_RESULT(yes); need_wrap_fcntl=no; AC_DEFINE(HAVE_LARGE_FILE_SUPPORT, 1,[Define this symbol if LFS is enabled]) ], [ AC_MSG_RESULT(no); need_wrap_fcntl=yes] ) # The following macro will add the necessary defines to bitcoin-config.h, but # they also need to be passed down to any subprojects. Pull the results out of # the cache and add them to CPPFLAGS. AC_SYS_LARGEFILE # detect POSIX or GNU variant of strerror_r AC_FUNC_STRERROR_R if test x$ac_cv_sys_file_offset_bits != x && test x$ac_cv_sys_file_offset_bits != xno && test x$ac_cv_sys_file_offset_bits != xunknown; then CPPFLAGS="$CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits" fi if test x$ac_cv_sys_large_files != x && test x$ac_cv_sys_large_files != xno && test x$ac_cv_sys_large_files != xunknown; then CPPFLAGS="$CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi AX_CHECK_LINK_FLAG([[-Wl,--large-address-aware]], [LDFLAGS="$LDFLAGS -Wl,--large-address-aware"]) AX_GCC_FUNC_ATTRIBUTE([visibility]) AX_GCC_FUNC_ATTRIBUTE([dllexport]) AX_GCC_FUNC_ATTRIBUTE([dllimport]) if test x$use_glibc_compat != xno; then #glibc absorbed clock_gettime in 2.17. librt (its previous location) is safe to link #in anyway for back-compat. AC_CHECK_LIB([rt],[clock_gettime],, AC_MSG_ERROR(librt missing)) #__fdelt_chk's params and return type have changed from long unsigned int to long int. # See which one is present here. AC_MSG_CHECKING(__fdelt_chk type) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#ifdef _FORTIFY_SOURCE #undef _FORTIFY_SOURCE #endif #define _FORTIFY_SOURCE 2 #include extern "C" long unsigned int __fdelt_warn(long unsigned int);]],[[]])], [ fdelt_type="long unsigned int"], [ fdelt_type="long int"]) AC_MSG_RESULT($fdelt_type) AC_DEFINE_UNQUOTED(FDELT_TYPE, $fdelt_type,[parameter and return value type for __fdelt_chk]) AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"]) AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"]) if test x$need_wrap_fcntl = xyes; then AX_CHECK_LINK_FLAG([[-Wl,--wrap=fcntl]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=fcntl"]) AX_CHECK_LINK_FLAG([[-Wl,--wrap=fcntl64]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=fcntl64"]) fi else AC_SEARCH_LIBS([clock_gettime],[rt]) fi if test x$TARGET_OS != xwindows; then # All windows code is PIC, forcing it on just adds useless compile warnings AX_CHECK_COMPILE_FLAG([-fPIC],[PIC_FLAGS="-fPIC"]) fi if test x$use_hardening != xno; then AX_CHECK_COMPILE_FLAG([-Wstack-protector],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) AX_CHECK_COMPILE_FLAG([-fstack-protector-all],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) AX_CHECK_PREPROC_FLAG([-D_FORTIFY_SOURCE=2],[ AX_CHECK_PREPROC_FLAG([-U_FORTIFY_SOURCE],[ HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -U_FORTIFY_SOURCE" ]) HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -D_FORTIFY_SOURCE=2" ]) AX_CHECK_LINK_FLAG([[-Wl,--dynamicbase]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"]) AX_CHECK_LINK_FLAG([[-Wl,--nxcompat]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"]) AX_CHECK_LINK_FLAG([[-Wl,--high-entropy-va]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"]) AX_CHECK_LINK_FLAG([[-Wl,-z,relro]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,relro"]) AX_CHECK_LINK_FLAG([[-Wl,-z,now]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"]) if test x$TARGET_OS != xwindows; then AX_CHECK_COMPILE_FLAG([-fPIE],[PIE_FLAGS="-fPIE"]) AX_CHECK_LINK_FLAG([[-pie]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -pie"]) fi case $host in *mingw*) AC_CHECK_LIB([ssp], [main],, AC_MSG_ERROR(libssp missing)) ;; esac fi dnl this flag screws up non-darwin gcc even when the check fails. special-case it. if test x$TARGET_OS = xdarwin; then AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"]) fi AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h]) AC_CHECK_DECLS([getifaddrs, freeifaddrs],,, [#include #include ] ) AC_CHECK_DECLS([strnlen]) # Check for daemon(3), unrelated to --with-daemon (although used by it) AC_CHECK_DECLS([daemon]) AC_CHECK_DECLS([le16toh, le32toh, le64toh, htole16, htole32, htole64, be16toh, be32toh, be64toh, htobe16, htobe32, htobe64],,, [#if HAVE_ENDIAN_H #include #elif HAVE_SYS_ENDIAN_H #include #endif]) AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,, [#if HAVE_BYTESWAP_H #include #endif]) AC_CHECK_DECLS([__builtin_clz, __builtin_clzl, __builtin_clzll, __builtin_popcount]) dnl Check for mallopt(M_ARENA_MAX) (to set glibc arenas) AC_MSG_CHECKING(for mallopt M_ARENA_MAX) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ mallopt(M_ARENA_MAX, 1); ]])], [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MALLOPT_ARENA_MAX, 1,[Define this symbol if you have mallopt with M_ARENA_MAX]) ], [ AC_MSG_RESULT(no)] ) dnl Check for malloc_info (for memory statistics information in getmemoryinfo) AC_MSG_CHECKING(for getmemoryinfo) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ int f = malloc_info(0, NULL); ]])], [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MALLOC_INFO, 1,[Define this symbol if you have malloc_info]) ], [ AC_MSG_RESULT(no)] ) AC_MSG_CHECKING([for visibility attribute]) AC_LINK_IFELSE([AC_LANG_SOURCE([ int foo_def( void ) __attribute__((visibility("default"))); int main(){} ])], [ AC_DEFINE(HAVE_VISIBILITY_ATTRIBUTE,1,[Define if the visibility attribute is supported.]) AC_MSG_RESULT(yes) ], [ AC_MSG_RESULT(no) if test x$use_reduce_exports = xyes; then AC_MSG_ERROR([Cannot find a working visibility attribute. Use --disable-reduce-exports.]) fi ] ) # Check for different ways of gathering OS randomness AC_MSG_CHECKING(for Linux getrandom syscall) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include #include ]], [[ syscall(SYS_getrandom, nullptr, 32, 0); ]])], [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYS_GETRANDOM, 1,[Define this symbol if the Linux getrandom system call is available]) ], [ AC_MSG_RESULT(no)] ) AC_MSG_CHECKING(for getentropy) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ getentropy(nullptr, 32) ]])], [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GETENTROPY, 1,[Define this symbol if the BSD getentropy system call is available]) ], [ AC_MSG_RESULT(no)] ) AC_MSG_CHECKING(for getentropy via random.h) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ getentropy(nullptr, 32) ]])], [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GETENTROPY_RAND, 1,[Define this symbol if the BSD getentropy system call is available with sys/random.h]) ], [ AC_MSG_RESULT(no)] ) AC_MSG_CHECKING(for sysctl KERN_ARND) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ static const int name[2] = {CTL_KERN, KERN_ARND}; sysctl(name, 2, nullptr, nullptr, nullptr, 0); ]])], [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL_ARND, 1,[Define this symbol if the BSD sysctl(KERN_ARND) is available]) ], [ AC_MSG_RESULT(no)] ) AC_MSG_CHECKING(for if type char equals int8_t) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ static_assert(std::is_same::value, ""); ]])], [ AC_MSG_RESULT(yes); AC_DEFINE(CHAR_EQUALS_INT8, 1,[Define this symbol if type char equals int8_t]) ], [ AC_MSG_RESULT(no)] ) # Check for reduced exports if test x$use_reduce_exports = xyes; then AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[RE_CXXFLAGS="-fvisibility=hidden"], [AC_MSG_ERROR([Cannot set default symbol visibility. Use --disable-reduce-exports.])]) fi LEVELDB_CPPFLAGS= LIBLEVELDB= LIBMEMENV= AM_CONDITIONAL([EMBEDDED_LEVELDB],[true]) AC_SUBST(LEVELDB_CPPFLAGS) AC_SUBST(LIBLEVELDB) AC_SUBST(LIBMEMENV) if test x$enable_wallet != xno; then dnl Check for libdb_cxx only if wallet enabled BITCOIN_FIND_BDB53 fi dnl Check for libminiupnpc (optional) if test x$use_upnp != xno; then AC_CHECK_HEADERS( [miniupnpc/miniwget.h miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h], [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS=-lminiupnpc], [have_miniupnpc=no])], [have_miniupnpc=no] ) fi BITCOIN_QT_INIT dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus BITCOIN_QT_CONFIGURE([$use_pkgconfig]) if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$build_bitcoin_seeder$use_tests$use_bench = xnononononono; then use_boost=no else use_boost=yes fi if test x$use_boost = xyes; then dnl Minimum required Boost version define(MINIMUM_REQUIRED_BOOST, 1.58.0) dnl Check for boost libs AX_BOOST_BASE([MINIMUM_REQUIRED_BOOST]) if test x$want_boost = xno; then AC_MSG_ERROR([[only libbitcoinconsensus can be built without boost]]) fi AX_BOOST_SYSTEM AX_BOOST_FILESYSTEM AX_BOOST_THREAD AX_BOOST_CHRONO dnl Boost 1.56 through 1.62 allow using std::atomic instead of its own atomic dnl counter implementations. In 1.63 and later the std::atomic approach is default. m4_pattern_allow(DBOOST_AC_USE_STD_ATOMIC) dnl otherwise it's treated like a macro BOOST_CPPFLAGS="-DBOOST_SP_USE_STD_ATOMIC -DBOOST_AC_USE_STD_ATOMIC $BOOST_CPPFLAGS" fi if test x$use_reduce_exports = xyes; then CXXFLAGS="$CXXFLAGS $RE_CXXFLAGS" AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]], [RELDFLAGS="-Wl,--exclude-libs,ALL"]) fi if test x$use_tests = xyes; then if test x$HEXDUMP = x; then AC_MSG_ERROR(hexdump is required for tests) fi if test x$use_boost = xyes; then AX_BOOST_UNIT_TEST_FRAMEWORK dnl Determine if -DBOOST_TEST_DYN_LINK is needed AC_MSG_CHECKING([for dynamic linked boost test]) TEMP_LIBS="$LIBS" LIBS="$LIBS $BOOST_LDFLAGS $BOOST_UNIT_TEST_FRAMEWORK_LIB" TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" AC_LINK_IFELSE([AC_LANG_SOURCE([ #define BOOST_TEST_DYN_LINK #define BOOST_TEST_MAIN #include ])], [AC_MSG_RESULT(yes)] [TESTDEFS="$TESTDEFS -DBOOST_TEST_DYN_LINK"], [AC_MSG_RESULT(no)]) LIBS="$TEMP_LIBS" CPPFLAGS="$TEMP_CPPFLAGS" fi fi if test x$use_boost = xyes; then BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_THREAD_LIB $BOOST_CHRONO_LIB" fi if test x$use_pkgconfig = xyes; then : dnl m4_ifdef( [PKG_CHECK_MODULES], [ PKG_CHECK_MODULES([SSL], [libssl],, [AC_MSG_ERROR(openssl not found.)]) PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto not found.)]) BITCOIN_QT_CHECK([PKG_CHECK_MODULES([PROTOBUF], [protobuf], [have_protobuf=yes], [BITCOIN_QT_FAIL(libprotobuf not found)])]) if test x$use_qr != xno; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then PKG_CHECK_MODULES([EVENT], [libevent],, [AC_MSG_ERROR(libevent not found.)]) if test x$TARGET_OS != xwindows; then PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads],, [AC_MSG_ERROR(libevent_pthreads not found.)]) fi fi if test "x$use_zmq" = "xyes"; then PKG_CHECK_MODULES([ZMQ],[libzmq >= 4], [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], [AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) use_zmq=no]) else AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) fi ] ) else AC_CHECK_HEADER([openssl/crypto.h],,AC_MSG_ERROR(libcrypto headers missing)) AC_CHECK_LIB([crypto], [main],CRYPTO_LIBS=-lcrypto, AC_MSG_ERROR(libcrypto missing)) AC_CHECK_HEADER([openssl/ssl.h],, AC_MSG_ERROR(libssl headers missing),) AC_CHECK_LIB([ssl], [main],SSL_LIBS=-lssl, AC_MSG_ERROR(libssl missing)) if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),) AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing)) if test x$TARGET_OS != xwindows; then AC_CHECK_LIB([event_pthreads],[main],EVENT_PTHREADS_LIBS=-levent_pthreads,AC_MSG_ERROR(libevent_pthreads missing)) fi fi if test "x$use_zmq" = "xyes"; then AC_CHECK_HEADER([zmq.h], [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], [AC_MSG_WARN([zmq.h not found, disabling zmq support]) use_zmq=no AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) AC_CHECK_LIB([zmq],[zmq_ctx_shutdown],ZMQ_LIBS=-lzmq, [AC_MSG_WARN([libzmq >= 4.0 not found, disabling zmq support]) use_zmq=no AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) else AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) fi if test "x$use_zmq" = "xyes"; then dnl Assume libzmq was built for static linking case $host in *mingw*) ZMQ_CFLAGS="$ZMQ_CFLAGS -DZMQ_STATIC" ;; esac fi BITCOIN_QT_CHECK(AC_CHECK_LIB([protobuf] ,[main],[PROTOBUF_LIBS=-lprotobuf], BITCOIN_QT_FAIL(libprotobuf not found))) if test x$use_qr != xno; then BITCOIN_QT_CHECK([AC_CHECK_LIB([qrencode], [main],[QR_LIBS=-lqrencode], [have_qrencode=no])]) BITCOIN_QT_CHECK([AC_CHECK_HEADER([qrencode.h],, have_qrencode=no)]) fi fi save_CXXFLAGS="${CXXFLAGS}" CXXFLAGS="${CXXFLAGS} ${CRYPTO_CFLAGS} ${SSL_CFLAGS}" AC_CHECK_DECLS([EVP_MD_CTX_new],,,[AC_INCLUDES_DEFAULT #include ]) CXXFLAGS="${save_CXXFLAGS}" dnl univalue check need_bundled_univalue=yes if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononono; then need_bundled_univalue=no else if test x$system_univalue != xno ; then found_univalue=no if test x$use_pkgconfig = xyes; then : #NOP m4_ifdef( [PKG_CHECK_MODULES], [ PKG_CHECK_MODULES([UNIVALUE],[libunivalue >= 1.0.4],[found_univalue=yes],[true]) ] ) else AC_CHECK_HEADER([univalue.h],[ AC_CHECK_LIB([univalue], [main],[ UNIVALUE_LIBS=-lunivalue found_univalue=yes ],[true]) ],[true]) fi if test x$found_univalue = xyes ; then system_univalue=yes need_bundled_univalue=no elif test x$system_univalue = xyes ; then AC_MSG_ERROR([univalue not found]) else system_univalue=no fi fi if test x$need_bundled_univalue = xyes ; then UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' UNIVALUE_LIBS='univalue/libunivalue.la' fi fi AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) AC_SUBST(UNIVALUE_CFLAGS) AC_SUBST(UNIVALUE_LIBS) BITCOIN_QT_PATH_PROGS([PROTOC], [protoc],$protoc_bin_path) AC_MSG_CHECKING([whether to build bitcoind]) AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes]) AC_MSG_RESULT($build_bitcoind) AC_MSG_CHECKING([whether to build bitcoin-seeder]) AM_CONDITIONAL([BUILD_BITCOIN_SEEDER], [test x$build_bitcoin_seeder = xyes]) AC_MSG_RESULT($build_bitcoin_seeder) AC_MSG_CHECKING([whether to build utils (bitcoin-cli bitcoin-tx)]) AM_CONDITIONAL([BUILD_BITCOIN_UTILS], [test x$build_bitcoin_utils = xyes]) AC_MSG_RESULT($build_bitcoin_utils) AC_MSG_CHECKING([whether to build libraries]) AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test x$build_bitcoin_libs = xyes]) if test x$build_bitcoin_libs = xyes; then AC_DEFINE(HAVE_CONSENSUS_LIB, 1, [Define this symbol if the consensus lib has been built]) AC_CONFIG_FILES([libbitcoinconsensus.pc:libbitcoinconsensus.pc.in]) fi AC_MSG_RESULT($build_bitcoin_libs) AC_LANG_POP if test "x$use_ccache" != "xno"; then AC_MSG_CHECKING(if ccache should be used) if test x$CCACHE = x; then if test "x$use_ccache" = "xyes"; then AC_MSG_ERROR([ccache not found.]); else use_ccache=no fi else use_ccache=yes CC="$ac_cv_path_CCACHE $CC" CXX="$ac_cv_path_CCACHE $CXX" fi AC_MSG_RESULT($use_ccache) fi if test "x$use_ccache" = "xyes"; then AX_CHECK_PREPROC_FLAG([-Qunused-arguments],[CPPFLAGS="-Qunused-arguments $CPPFLAGS"]) fi dnl enable wallet AC_MSG_CHECKING([if wallet should be enabled]) if test x$enable_wallet != xno; then AC_MSG_RESULT(yes) AC_DEFINE_UNQUOTED([ENABLE_WALLET],[1],[Define to 1 to enable wallet functions]) else AC_MSG_RESULT(no) fi dnl enable upnp support AC_MSG_CHECKING([whether to build with support for UPnP]) if test x$have_miniupnpc = xno; then if test x$use_upnp = xyes; then AC_MSG_ERROR("UPnP requested but cannot be built. use --without-miniupnpc") fi AC_MSG_RESULT(no) else if test x$use_upnp != xno; then AC_MSG_RESULT(yes) AC_MSG_CHECKING([whether to build with UPnP enabled by default]) use_upnp=yes upnp_setting=0 if test x$use_upnp_default != xno; then use_upnp_default=yes upnp_setting=1 fi AC_MSG_RESULT($use_upnp_default) AC_DEFINE_UNQUOTED([USE_UPNP],[$upnp_setting],[UPnP support not compiled if undefined, otherwise value (0 or 1) determines default state]) if test x$TARGET_OS = xwindows; then MINIUPNPC_CPPFLAGS="-DSTATICLIB -DMINIUPNP_STATICLIB" fi else AC_MSG_RESULT(no) fi fi dnl these are only used when qt is enabled BUILD_TEST_QT="" if test x$bitcoin_enable_qt != xno; then dnl enable dbus support AC_MSG_CHECKING([whether to build GUI with support for D-Bus]) if test x$bitcoin_enable_qt_dbus != xno; then AC_DEFINE([USE_DBUS],[1],[Define if dbus support should be compiled in]) fi AC_MSG_RESULT($bitcoin_enable_qt_dbus) dnl enable qr support AC_MSG_CHECKING([whether to build GUI with support for QR codes]) if test x$have_qrencode = xno; then if test x$use_qr = xyes; then AC_MSG_ERROR("QR support requested but cannot be built. use --without-qrencode") fi AC_MSG_RESULT(no) else if test x$use_qr != xno; then AC_MSG_RESULT(yes) AC_DEFINE([USE_QRCODE],[1],[Define if QR support should be compiled in]) use_qr=yes else AC_MSG_RESULT(no) fi fi if test x$XGETTEXT = x; then AC_MSG_WARN("xgettext is required to update qt translations") fi AC_MSG_CHECKING([whether to build test_bitcoin-qt]) if test x$use_gui_tests$bitcoin_enable_qt_test = xyesyes; then AC_MSG_RESULT([yes]) BUILD_TEST_QT="yes" else AC_MSG_RESULT([no]) fi fi AM_CONDITIONAL([ENABLE_ZMQ], [test "x$use_zmq" = "xyes"]) AC_MSG_CHECKING([whether to build test_bitcoin]) if test x$use_tests = xyes; then AC_MSG_RESULT([yes]) BUILD_TEST="yes" else AC_MSG_RESULT([no]) BUILD_TEST="" fi AC_MSG_CHECKING([whether to reduce exports]) if test x$use_reduce_exports = xyes; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) fi if test x$build_bitcoin_utils$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests = xnononononono; then AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-seeder --with-gui --enable-bench or --enable-tests]) fi AM_CONDITIONAL([TARGET_DARWIN], [test x$TARGET_OS = xdarwin]) AM_CONDITIONAL([BUILD_DARWIN], [test x$BUILD_OS = xdarwin]) AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows]) AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes]) AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes]) AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes]) AM_CONDITIONAL([ENABLE_HWCRC32],[test x$enable_hwcrc32 = xyes]) AM_CONDITIONAL([ENABLE_SSE41],[test x$enable_sse41 = xyes]) AM_CONDITIONAL([ENABLE_AVX2],[test x$enable_avx2 = xyes]) AM_CONDITIONAL([ENABLE_SHANI],[test x$enable_shani = xyes]) AM_CONDITIONAL([USE_ASM],[test x$use_asm = xyes]) AC_DEFINE(COPYRIGHT_YEAR, _COPYRIGHT_YEAR, [Copyright year]) AC_DEFINE(COPYRIGHT_HOLDERS, "_COPYRIGHT_HOLDERS", [Copyright holder(s) before %s replacement]) AC_DEFINE(COPYRIGHT_HOLDERS_SUBSTITUTION, "_COPYRIGHT_HOLDERS_SUBSTITUTION", [Replacement for %s in copyright holders string]) define(_COPYRIGHT_HOLDERS_FINAL, [patsubst(_COPYRIGHT_HOLDERS, [%s], [_COPYRIGHT_HOLDERS_SUBSTITUTION])]) AC_DEFINE(COPYRIGHT_HOLDERS_FINAL, "_COPYRIGHT_HOLDERS_FINAL", [Copyright holder(s)]) AC_SUBST(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR) AC_SUBST(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR) AC_SUBST(CLIENT_VERSION_REVISION, _CLIENT_VERSION_REVISION) AC_SUBST(CLIENT_VERSION_IS_RELEASE, _CLIENT_VERSION_IS_RELEASE) AC_SUBST(COPYRIGHT_YEAR, _COPYRIGHT_YEAR) AC_SUBST(COPYRIGHT_HOLDERS, "_COPYRIGHT_HOLDERS") AC_SUBST(COPYRIGHT_HOLDERS_SUBSTITUTION, "_COPYRIGHT_HOLDERS_SUBSTITUTION") AC_SUBST(COPYRIGHT_HOLDERS_FINAL, "_COPYRIGHT_HOLDERS_FINAL") AC_SUBST(BITCOIN_DAEMON_NAME) AC_SUBST(BITCOIN_GUI_NAME) AC_SUBST(BITCOIN_CLI_NAME) AC_SUBST(BITCOIN_TX_NAME) AC_SUBST(BITCOIN_SEEDER_NAME) AC_SUBST(RELDFLAGS) AC_SUBST(DEBUG_CPPFLAGS) AC_SUBST(WARN_CXXFLAGS) AC_SUBST(NOWARN_CXXFLAGS) AC_SUBST(DEBUG_CXXFLAGS) AC_SUBST(COMPAT_LDFLAGS) AC_SUBST(ERROR_CXXFLAGS) AC_SUBST(HARDENED_CXXFLAGS) AC_SUBST(HARDENED_CPPFLAGS) AC_SUBST(HARDENED_LDFLAGS) AC_SUBST(PIC_FLAGS) AC_SUBST(PIE_FLAGS) AC_SUBST(SANITIZER_CXXFLAGS) AC_SUBST(SANITIZER_LDFLAGS) AC_SUBST(SSE42_CXXFLAGS) AC_SUBST(SSE41_CXXFLAGS) AC_SUBST(AVX2_CXXFLAGS) AC_SUBST(SHANI_CXXFLAGS) AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(USE_UPNP) AC_SUBST(USE_QRCODE) AC_SUBST(BOOST_LIBS) AC_SUBST(TESTDEFS) AC_SUBST(LEVELDB_TARGET_FLAGS) AC_SUBST(MINIUPNPC_CPPFLAGS) AC_SUBST(MINIUPNPC_LIBS) AC_SUBST(CRYPTO_LIBS) AC_SUBST(SSL_LIBS) AC_SUBST(EVENT_LIBS) AC_SUBST(EVENT_PTHREADS_LIBS) AC_SUBST(ZMQ_LIBS) AC_SUBST(PROTOBUF_LIBS) AC_SUBST(QR_LIBS) AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) AC_CONFIG_FILES([doc/Doxyfile]) AC_CONFIG_FILES([src/config/version.h]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/util/bitcoin-util-test.py:test/util/bitcoin-util-test.py]) AC_CONFIG_LINKS([test/util/rpcauth-test.py:test/util/rpcauth-test.py]) dnl boost's m4 checks do something really nasty: they export these vars. As a dnl result, they leak into secp256k1's configure and crazy things happen. dnl Until this is fixed upstream and we've synced, we'll just un-export them. CPPFLAGS_TEMP="$CPPFLAGS" unset CPPFLAGS CPPFLAGS="$CPPFLAGS_TEMP" LDFLAGS_TEMP="$LDFLAGS" unset LDFLAGS LDFLAGS="$LDFLAGS_TEMP" LIBS_TEMP="$LIBS" unset LIBS LIBS="$LIBS_TEMP" PKGCONFIG_PATH_TEMP="$PKG_CONFIG_PATH" unset PKG_CONFIG_PATH PKG_CONFIG_PATH="$PKGCONFIG_PATH_TEMP" PKGCONFIG_LIBDIR_TEMP="$PKG_CONFIG_LIBDIR" unset PKG_CONFIG_LIBDIR PKG_CONFIG_LIBDIR="$PKGCONFIG_LIBDIR_TEMP" if test x$need_bundled_univalue = xyes; then AC_CONFIG_SUBDIRS([src/univalue]) fi ac_configure_args="${ac_configure_args} --disable-shared --with-pic --with-bignum=no --enable-module-recovery --enable-module-multiset --disable-jni" AC_CONFIG_SUBDIRS([src/secp256k1]) AC_OUTPUT dnl Taken from https://wiki.debian.org/RpathIssue case $host in *-*-linux-gnu) AC_MSG_RESULT([Fixing libtool for -rpath problems.]) sed < libtool > libtool-2 \ 's/^hardcode_libdir_flag_spec.*$'/'hardcode_libdir_flag_spec=" -D__LIBTOOL_IS_A_FOOL__ "/' mv libtool-2 libtool chmod 755 libtool ;; esac dnl Replace the BUILDDIR path with the correct Windows path if compiling on Native Windows case ${OS} in *Windows*) sed 's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/' test/config.ini > test/config-2.ini mv test/config-2.ini test/config.ini ;; esac echo echo "Options used to compile and link:" echo " with wallet = $enable_wallet" echo " with gui / qt = $bitcoin_enable_qt" if test x$bitcoin_enable_qt != xno; then echo " with qr = $use_qr" fi echo " with zmq = $use_zmq" echo " with test = $use_tests" echo " with bench = $use_bench" echo " with upnp = $use_upnp" echo " use asm = $use_asm" echo " sanitizers = $use_sanitizers" echo " debug enabled = $enable_debug" echo " werror = $enable_werror" echo echo " target os = $TARGET_OS" echo " build os = $BUILD_OS" echo echo " CC = $CC" echo " CFLAGS = $CFLAGS" echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" echo " CXXFLAGS = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $CXXFLAGS" echo " LDFLAGS = $PTHREAD_CFLAGS $HARDENED_LDFLAGS $LDFLAGS" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 534d5470a..3d0171e1b 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -1,130 +1,130 @@ Contents ======== This directory contains tools for developers working on this repository. copyright\_header.py ==================== Provides utilities for managing copyright headers of `The Bitcoin developers` in repository source files. It has three subcommands: ``` $ ./copyright_header.py report [verbose] $ ./copyright_header.py update $ ./copyright_header.py insert ``` Running these subcommands without arguments displays a usage string. copyright\_header.py report \ [verbose] --------------------------------------------------------- Produces a report of all copyright header notices found inside the source files of a repository. Useful to quickly visualize the state of the headers. Specifying `verbose` will list the full filenames of files of each category. copyright\_header.py update \ [verbose] --------------------------------------------------------- Updates all the copyright headers of `The Bitcoin developers` which were changed in a year more recent than is listed. For example: ``` // Copyright (c) - The Bitcoin developers ``` will be updated to: ``` // Copyright (c) - The Bitcoin developers ``` where `` is obtained from the `git log` history. This subcommand also handles copyright headers that have only a single year. In those cases: ``` // Copyright (c) The Bitcoin developers ``` will be updated to: ``` // Copyright (c) - The Bitcoin developers ``` where the update is appropriate. copyright\_header.py insert \ ------------------------------------ Inserts a copyright header for `The Bitcoin developers` at the top of the file in either Python or C++ style as determined by the file extension. If the file is a Python file and it has `#!` starting the first line, the header is inserted in the line below it. The copyright dates will be set to be `-` where `` is according to the `git log` history. If `` is equal to ``, it will be set as a single year rather than two hyphenated years. If the file already has a copyright for `The Bitcoin developers`, the script will exit. gen-manpages.sh =============== A small script to automatically create manpages in ../../doc/man by running the release binaries with the -help option. This requires help2man which can be found at: https://www.gnu.org/software/help2man/ With in-tree builds this tool can be run from any directory within the repostitory. To use this tool with out-of-tree builds set `BUILDDIR`. For example: ```bash BUILDDIR=$PWD/build contrib/devtools/gen-manpages.sh ``` optimize-pngs.py ================ A script to optimize png files in the bitcoin repository (requires pngcrush). security-check.py and test-security-check.py ============================================ Perform basic ELF security checks on a series of executables. symbol-check.py =============== A script to check that the (Linux) executables produced by gitian only contain allowed gcc, glibc and libstdc++ version symbols. This makes sure they are still compatible with the minimum supported Linux distribution versions. Example usage after a gitian build: - find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py + find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py If only supported symbols are used the return value will be 0 and the output will be empty. If there are 'unsupported' symbols, the return value will be 1 a list like this will be printed: .../64/test_bitcoin: symbol memcpy from unsupported version GLIBC_2.14 .../64/test_bitcoin: symbol __fdelt_chk from unsupported version GLIBC_2.15 .../64/test_bitcoin: symbol std::out_of_range::~out_of_range() from unsupported version GLIBCXX_3.4.15 .../64/test_bitcoin: symbol _ZNSt8__detail15_List_nod from unsupported version GLIBCXX_3.4.15 update-translations.py ====================== Run this script from the root of the repository to update all translations from transifex. It will do the following automatically: - Fetch all translations - Post-process them into valid and committable format - Add missing translations to the build system (TODO) See doc/translation-process.md for more information. circular-dependencies.py ======================== Run this script from the root of the source tree (`src/`) to find circular dependencies in the source code. This looks only at which files include other files, treating the `.cpp` and `.h` file as one unit. Example usage: cd /src ../contrib/devtools/circular-dependencies.py {*,*/*,*/*/*}.{h,cpp} diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index c6b2649aa..dc961e974 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -1,190 +1,190 @@ #!/usr/bin/env python3 # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' A script to check that the (Linux) executables produced by gitian only contain allowed gcc, glibc and libstdc++ version symbols. This makes sure they are still compatible with the minimum supported Linux distribution versions. Example usage: - find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py + find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' import subprocess import re import sys import os # Debian 8.11 (Jessie) has: # # - g++ version 4.9.2 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=g%2B%2B) # - libc version 2.19.18 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libc6) # - libstdc++ version 4.8.4 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libstdc%2B%2B6) # # Ubuntu 14.04 (Trusty Tahr) has: # # - g++ version 4.8.2 (https://packages.ubuntu.com/search?suite=trusty§ion=all&arch=any&keywords=g%2B%2B&searchon=names) # - libc version 2.19.0 (https://packages.ubuntu.com/search?suite=trusty§ion=all&arch=any&keywords=libc6&searchon=names) # - libstdc++ version 4.8.2 (https://packages.ubuntu.com/search?suite=trusty§ion=all&arch=any&keywords=libstdc%2B%2B&searchon=names) # # Taking the minimum of these as our target. # # According to GNU ABI document (http://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: # GCC 4.8.0: GCC_4.8.0 # GCC 4.8.0: GLIBCXX_3.4.18, CXXABI_1.3.7 # (glibc) GLIBC_2_19 # MAX_VERSIONS = { 'GCC': (4, 8, 0), 'CXXABI': (1, 3, 7), 'GLIBCXX': (3, 4, 18), 'GLIBC': (2, 19), 'LIBATOMIC': (1, 0) } # See here for a description of _IO_stdin_used: # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { '_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', # Figure out why we get these symbols exported on xenial. '_ZNKSt5ctypeIcE8do_widenEc', 'in6addr_any', 'optarg', '_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_destroyEv' } READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') # Allowed NEEDED libraries ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt 'libgcc_s.so.1', # GCC base support 'libc.so.6', # C library 'libpthread.so.0', # threading 'libanl.so.1', # DNS resolve 'libm.so.6', # math library 'librt.so.1', # real-time (clock) 'libatomic.so.1', 'ld-linux-x86-64.so.2', # 64-bit dynamic linker 'ld-linux.so.2', # 32-bit dynamic linker 'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker 'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker # bitcoin-qt only 'libX11-xcb.so.1', # part of X11 'libX11.so.6', # part of X11 'libxcb.so.1', # part of X11 'libfontconfig.so.1', # font support 'libfreetype.so.6', # font parsing 'libdl.so.2' # programming interface to dynamic linker } ARCH_MIN_GLIBC_VER = { '80386': (2, 1), 'X86-64': (2, 2, 5), 'ARM': (2, 4), 'AArch64': (2, 17) } class CPPFilt(object): ''' Demangle C++ symbol names. Use a pipe to the 'c++filt' command. ''' def __init__(self): self.proc = subprocess.Popen( CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) def __call__(self, mangled): self.proc.stdin.write(mangled + '\n') self.proc.stdin.flush() return self.proc.stdout.readline().rstrip() def close(self): self.proc.stdin.close() self.proc.stdout.close() self.proc.wait() def read_symbols(executable, imports=True): ''' Parse an ELF executable and return a list of (symbol,version) tuples for dynamic, imported symbols. ''' p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', '-h', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Could not read symbols for {}: {}'.format( executable, stderr.strip())) syms = [] for line in stdout.splitlines(): line = line.split() if 'Machine:' in line: arch = line[-1] if len(line) > 7 and re.match('[0-9]+:$', line[0]): (sym, _, version) = line[7].partition('@') is_import = line[6] == 'UND' if version.startswith('@'): version = version[1:] if is_import == imports: syms.append((sym, version, arch)) return syms def check_version(max_versions, version, arch): if '_' in version: (lib, _, ver) = version.rpartition('_') else: lib = version ver = '0' ver = tuple([int(x) for x in ver.split('.')]) if not lib in max_versions: return False return ver <= max_versions[lib] or lib == 'GLIBC' and ver <= ARCH_MIN_GLIBC_VER[arch] def read_libraries(filename): p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') libraries = [] for line in stdout.splitlines(): tokens = line.split() if len(tokens) > 2 and tokens[1] == '(NEEDED)': match = re.match( r'^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) if match: libraries.append(match.group(1)) else: raise ValueError('Unparseable (NEEDED) specification') return libraries if __name__ == '__main__': cppfilt = CPPFilt() retval = 0 for filename in sys.argv[1:]: # Check imported symbols for sym, version, arch in read_symbols(filename, True): if version and not check_version(MAX_VERSIONS, version, arch): print('{}: symbol {} from unsupported version {}'.format( filename, cppfilt(sym), version)) retval = 1 # Check exported symbols for sym, version, arch in read_symbols(filename, False): if sym in IGNORE_EXPORTS: continue print('{}: export of symbol {} not allowed'.format( filename, cppfilt(sym))) retval = 1 # Check dependency libraries for library_name in read_libraries(filename): if library_name not in ALLOWED_LIBRARIES: print('{}: NEEDED library {} is not allowed'.format( filename, library_name)) retval = 1 sys.exit(retval) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 30d41713b..4adc62299 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,144 +1,144 @@ --- name: "bitcoin-abc-0.20-linux" enable_cache: true distro: "debian" suites: - "buster" architectures: - "amd64" multiarch: - "i386" packages: - "autoconf" - "automake" - "binutils-aarch64-linux-gnu" - "binutils-arm-linux-gnueabihf" - "binutils-gold" - "bsdmainutils" - "build-essential" - "ca-certificates" - "curl" - "faketime" - "g++-aarch64-linux-gnu" - "g++-arm-linux-gnueabihf" - "gcc-aarch64-linux-gnu" - "gcc-arm-linux-gnueabihf" - "git" - "gperf" - "lib32stdc++-8-dev" - "libc6-dev:i386" - "libtool" - "pkg-config" -- "python" +- "python3" remotes: - "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git" "dir": "bitcoin" files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu" CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="date ar ranlib nm" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" HOST_LDFLAGS=-static-libstdc++ export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export GZIP="-9n" export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" create_per-host_faketime_wrappers "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} cd bitcoin BASEPREFIX=`pwd`/depends # Build dependencies for each host for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done # Faketime for binaries export PATH=${PATH_orig} create_global_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} # Create the release tarball using (arbitrarily) the first host ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist SOURCEDIST=`echo bitcoin-abc-*.tar.gz` DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` # Correct tar file order mkdir -p temp pushd temp tar xf ../$SOURCEDIST find bitcoin-abc-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST popd ORIGPATH="$PATH" # Extract the release tarball into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH=`pwd`/installed/${DISTNAME} mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make ${MAKEOPTS} -C src check-symbols make install DESTDIR=${INSTALLPATH} cd installed find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig find ${DISTNAME}/bin -type f -executable -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; find ${DISTNAME}/lib -type f -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz cd ../../ rm -rf distsrc-${i} done mkdir -p $OUTDIR/src mv $SOURCEDIST $OUTDIR/src diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index c4dfe541a..ea76c80e6 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,162 +1,162 @@ --- name: "bitcoin-abc-0.20-osx" enable_cache: true distro: "debian" suites: - "buster" architectures: - "amd64" packages: - "autoconf" - "automake" - "bsdmainutils" - "ca-certificates" - "cmake" - "curl" - "faketime" - "fonts-tuffy" - "g++" - "git" - "imagemagick" - "libbz2-dev" - "libcap-dev" - "librsvg2-bin" - "libtiff-tools" - "libtinfo5" - "libtool" - "libz-dev" - "ninja-build" - "pkg-config" -- "python" -- "python-dev" -- "python-setuptools" +- "python3" +- "python3-dev" +- "python3-setuptools" remotes: - "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git" "dir": "bitcoin" files: - "MacOSX10.11.sdk.tar.gz" script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin14" CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="ar ranlib date dmg genisoimage" export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export GZIP="-9n" export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi export ZERO_AR_DATE=1 function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" create_per-host_faketime_wrappers "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} cd bitcoin BASEPREFIX=`pwd`/depends mkdir -p ${BASEPREFIX}/SDKs tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz # Build dependencies for each host for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done # Faketime for binaries export PATH=${PATH_orig} create_global_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} # Create the release tarball using (arbitrarily) the first host ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist SOURCEDIST=`echo bitcoin-abc-*.tar.gz` DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` # Correct tar file order mkdir -p temp pushd temp tar xf ../$SOURCEDIST find bitcoin-abc-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST popd ORIGPATH="$PATH" # Extract the release tarball into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH=`pwd`/installed/${DISTNAME} mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} make install-strip DESTDIR=${INSTALLPATH} make osx_volname make deploydir OSX_VOLNAME="$(cat osx_volname)" mkdir -p unsigned-app-${i} cp osx_volname unsigned-app-${i}/ cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i} cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff mv dist unsigned-app-${i} pushd unsigned-app-${i} find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz popd make deploy ${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg cd installed find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig find ${DISTNAME} | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz cd ../../ done mkdir -p $OUTDIR/src mv $SOURCEDIST $OUTDIR/src mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 18513a696..29f221f4e 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -1,171 +1,171 @@ --- name: "bitcoin-abc-0.20-win" enable_cache: true distro: "debian" suites: - "buster" architectures: - "amd64" packages: - "autoconf" - "automake" - "bsdmainutils" - "ca-certificates" - "curl" - "faketime" - "g++" - "g++-mingw-w64" - "git" - "libtool" - "mingw-w64" - "nsis" - "pkg-config" -- "python" +- "python3" - "rename" - "zip" remotes: - "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git" "dir": "bitcoin" files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --without-seeder" FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export GZIP="-9n" export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } function create_per-host_linker_wrapper { # This is only needed for trusty, as the mingw linker leaks a few bytes of # heap, causing non-determinism. See discussion in https://github.com/bitcoin/bitcoin/pull/6900 for i in $HOSTS; do mkdir -p ${WRAP_DIR}/${i} for prog in collect2; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}/${prog} REAL=$(${i}-gcc -print-prog-name=${prog}) echo "export MALLOC_PERTURB_=255" >> ${WRAP_DIR}/${i}/${prog} echo "${REAL} \$@" >> $WRAP_DIR/${i}/${prog} chmod +x ${WRAP_DIR}/${i}/${prog} done for prog in gcc g++; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" create_per-host_faketime_wrappers "2000-01-01 12:00:00" create_per-host_linker_wrapper "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} cd bitcoin BASEPREFIX=`pwd`/depends # Build dependencies for each host for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done # Faketime for binaries export PATH=${PATH_orig} create_global_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_linker_wrapper "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} # Create the release tarball using (arbitrarily) the first host ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist SOURCEDIST=`echo bitcoin-abc-*.tar.gz` DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` # Correct tar file order mkdir -p temp pushd temp tar xf ../$SOURCEDIST find bitcoin-abc-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST mkdir -p $OUTDIR/src cp ../$SOURCEDIST $OUTDIR/src popd ORIGPATH="$PATH" # Extract the release tarball into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH=`pwd`/installed/${DISTNAME} mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make deploy make install DESTDIR=${INSTALLPATH} cp -f bitcoin-abc-*setup*.exe $OUTDIR/ cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig find ${DISTNAME}/bin -type f -executable -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; find ${DISTNAME}/lib -type f -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}-debug.zip cd ../../ rm -rf distsrc-${i} done cd $OUTDIR rename 's/-setup\.exe$/-setup-unsigned.exe/' *-setup.exe find . -name "*-setup-unsigned.exe" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz mv ${OUTDIR}/${DISTNAME}-x86_64-*-debug.zip ${OUTDIR}/${DISTNAME}-win64-debug.zip mv ${OUTDIR}/${DISTNAME}-i686-*-debug.zip ${OUTDIR}/${DISTNAME}-win32-debug.zip mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip mv ${OUTDIR}/${DISTNAME}-i686-*.zip ${OUTDIR}/${DISTNAME}-win32.zip diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index 298510698..25a1c7351 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -1,55 +1,54 @@ # Linearize -Construct a linear, no-fork, best version of the Bitcoin blockchain. The scripts -run using Python 3 but are compatible with Python 2. +Construct a linear, no-fork, best version of the Bitcoin blockchain. ## Step 1: Download hash list $ ./linearize-hashes.py linearize.cfg > hashlist.txt Required configuration file settings for linearize-hashes: * RPC: `datadir` (Required if `rpcuser` and `rpcpassword` are not specified) * RPC: `rpcuser`, `rpcpassword` (Required if `datadir` is not specified) Optional config file setting for linearize-hashes: * RPC: `host` (Default: `127.0.0.1`) * RPC: `port` (Default: `8332`) * Blockchain: `min_height`, `max_height` * `rev_hash_bytes`: If true, the written block hash list will be byte-reversed. (In other words, the hash returned by getblockhash will have its bytes reversed.) False by default. Intended for generation of standalone hash lists but safe to use with linearize-data.py, which will output the same data no matter which byte format is chosen. The `linearize-hashes` script requires a connection, local or remote, to a JSON-RPC server. Running `bitcoind` or `bitcoin-qt -server` will be sufficient. ## Step 2: Copy local block data $ ./linearize-data.py linearize.cfg Required configuration file settings: * `output_file`: The file that will contain the final blockchain. or * `output`: Output directory for linearized `blocks/blkNNNNN.dat` output. Optional config file setting for linearize-data: * `debug_output`: Some printouts may not always be desired. If true, such output will be printed. * `file_timestamp`: Set each file's last-accessed and last-modified times, respectively, to the current time and to the timestamp of the most recent block written to the script's blockchain. * `genesis`: The hash of the genesis block in the blockchain. * `input`: bitcoind blocks/ directory containing blkNNNNN.dat * `hashlist`: text file containing list of block hashes created by linearize-hashes.py. * `max_out_sz`: Maximum size for files created by the `output_file` option. (Default: `1000*1000*1000 bytes`) * `netmagic`: Network magic number. * `out_of_order_cache_sz`: If out-of-order blocks are being read, the block can be written to a cache so that the blockchain doesn't have to be sought again. This option specifies the cache size. (Default: `100*1000*1000 bytes`) * `rev_hash_bytes`: If true, the block hash list written by linearize-hashes.py will be byte-reversed when read by linearize-data.py. See the linearize-hashes entry for more information. * `split_timestamp`: Split blockchain files when a new month is first seen, in addition to reaching a maximum file size (`max_out_sz`). diff --git a/doc/dependencies.md b/doc/dependencies.md index 8a8e0ca28..f86f114e6 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -1,31 +1,31 @@ Dependencies ============ These are the dependencies currently used by Bitcoin Core. You can find instructions for installing them in the `build-*.md` file for your platform. | Dependency | Version used | Minimum required | CVEs | Shared | [Bundled Qt library](https://doc.qt.io/qt-5/configure-options.html) | | --- | --- | --- | --- | --- | --- | | Berkeley DB | [5.3.28](http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 5.3 | No | | | | Boost | [1.64.0](http://www.boost.org/users/download/) | 1.58.0 | No | | | | ccache | [3.3.6](https://ccache.samba.org/download.html) | | No | | | | Clang | | [3.4+](http://llvm.org/releases/download.html) (C++14 support) | | | | | D-Bus | [1.10.18](https://cgit.freedesktop.org/dbus/dbus/tree/NEWS?h=dbus-1.10) | | No | Yes | | | Expat | [2.2.0](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.6](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | | FreeType | [2.7.1](http://download.savannah.gnu.org/releases/freetype) | | No | | | | GCC | | [5.0+](https://gcc.gnu.org/) (C++14 support) | | | | | HarfBuzz-NG | | | | | | | libevent | [2.1.8-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | | libjpeg | | | | | Yes | | libpng | | | | | Yes | | MiniUPnPc | [2.0.20170509](http://miniupnp.free.fr/files) | | No | | | | OpenSSL | [1.0.1k](https://www.openssl.org/source) | | Yes | | | | PCRE | | | | | Yes | | protobuf | [2.6.1](https://github.com/google/protobuf/releases) | | No | | | -| Python (tests) | | [3.4](https://www.python.org/downloads) | | | | +| Python (tests) | | [3.5](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | | Qt | [5.9.6](https://download.qt.io/official_releases/qt/) | 5.5.1 | No | | | | XCB | | | | | Yes (Linux only) | | xkbcommon | | | | | Yes (Linux only) | | ZeroMQ | [4.1.5](https://github.com/zeromq/libzmq/releases) | | No | | | | zlib | [1.2.11](http://zlib.net/) | | | | No | diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index e531b49c0..5f20b259d 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -1,633 +1,630 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik # Copyright (c) 2010-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin P2P network half-a-node. This python code was modified from ArtForz' public domain half-a-node, as found in the mini-node branch of http://github.com/jgarzik/pynode. P2PConnection: A low-level connection object to a node's P2P interface P2PInterface: A high-level interface object for communicating to a node over P2P P2PDataStore: A p2p interface class that keeps a store of transactions and blocks and can respond correctly to getdata and getheaders messages""" import asyncio from collections import defaultdict from io import BytesIO import logging import struct import sys import threading from test_framework.messages import ( CBlockHeader, MIN_VERSION_SUPPORTED, msg_addr, msg_block, MSG_BLOCK, msg_blocktxn, msg_cmpctblock, msg_feefilter, msg_getaddr, msg_getblocks, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_mempool, msg_notfound, msg_ping, msg_pong, msg_reject, msg_sendcmpct, msg_sendheaders, msg_tx, MSG_TX, MSG_TYPE_MASK, msg_verack, msg_version, NODE_NETWORK, sha256, ) from test_framework.util import wait_until logger = logging.getLogger("TestFramework.mininode") MESSAGEMAP = { b"addr": msg_addr, b"block": msg_block, b"blocktxn": msg_blocktxn, b"cmpctblock": msg_cmpctblock, b"feefilter": msg_feefilter, b"getaddr": msg_getaddr, b"getblocks": msg_getblocks, b"getblocktxn": msg_getblocktxn, b"getdata": msg_getdata, b"getheaders": msg_getheaders, b"headers": msg_headers, b"inv": msg_inv, b"mempool": msg_mempool, b"notfound": msg_notfound, b"ping": msg_ping, b"pong": msg_pong, b"reject": msg_reject, b"sendcmpct": msg_sendcmpct, b"sendheaders": msg_sendheaders, b"tx": msg_tx, b"verack": msg_verack, b"version": msg_version, } MAGIC_BYTES = { "mainnet": b"\xe3\xe1\xf3\xe8", "testnet3": b"\xf4\xe5\xf3\xf4", "regtest": b"\xda\xb5\xbf\xfa", } class P2PConnection(asyncio.Protocol): """A low-level connection object to a node's P2P interface. This class is responsible for: - opening and closing the TCP connection to the node - reading bytes from and writing bytes to the socket - deserializing and serializing the P2P message header - logging messages as they are sent and received This class contains no logic for handing the P2P message payloads. It must be sub-classed and the on_message() callback overridden.""" def __init__(self): # The underlying transport of the connection. # Should only call methods on this from the NetworkThread, c.f. call_soon_threadsafe self._transport = None @property def is_connected(self): return self._transport is not None def peer_connect(self, dstaddr, dstport, net="regtest"): assert not self.is_connected self.dstaddr = dstaddr self.dstport = dstport # The initial message to send after the connection was made: self.on_connection_send_msg = None self.on_connection_send_msg_is_raw = False self.recvbuf = b"" self.network = net logger.debug('Connecting to Bitcoin Node: {}:{}'.format( self.dstaddr, self.dstport)) loop = NetworkThread.network_event_loop conn_gen_unsafe = loop.create_connection( lambda: self, host=self.dstaddr, port=self.dstport) def conn_gen(): return loop.call_soon_threadsafe( loop.create_task, conn_gen_unsafe) return conn_gen def peer_disconnect(self): # Connection could have already been closed by other end. NetworkThread.network_event_loop.call_soon_threadsafe( lambda: self._transport and self._transport.abort()) # Connection and disconnection methods def connection_made(self, transport): """asyncio callback when a connection is opened.""" assert not self._transport logger.debug("Connected & Listening: {}:{}".format( self.dstaddr, self.dstport)) self._transport = transport if self.on_connection_send_msg: if self.on_connection_send_msg_is_raw: self.send_raw_message(self.on_connection_send_msg) else: self.send_message(self.on_connection_send_msg) # Never used again self.on_connection_send_msg = None self.on_open() def connection_lost(self, exc): """asyncio callback when a connection is closed.""" if exc: logger.warning("Connection lost to {}:{} due to {}".format( self.dstaddr, self.dstport, exc)) else: logger.debug("Closed connection to: {}:{}".format( self.dstaddr, self.dstport)) self._transport = None self.recvbuf = b"" self.on_close() # Socket read methods def data_received(self, t): """asyncio callback when data is read from the socket.""" with mininode_lock: if len(t) > 0: self.recvbuf += t while True: msg = self._on_data() if msg == None: break self.on_message(msg) def _on_data(self): """Try to read P2P messages from the recv buffer. This method reads data from the buffer in a loop. It deserializes, parses and verifies the P2P header, then passes the P2P payload to the on_message callback for processing.""" try: with mininode_lock: if len(self.recvbuf) < 4: return None if self.recvbuf[:4] != MAGIC_BYTES[self.network]: raise ValueError( "got garbage {}".format(repr(self.recvbuf))) if len(self.recvbuf) < 4 + 12 + 4 + 4: return None command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] msglen = struct.unpack(" 500: log_message += "... (msg truncated)" logger.debug(log_message) class P2PInterface(P2PConnection): """A high-level P2P interface class for communicating with a Bitcoin Cash node. This class provides high-level callbacks for processing P2P message payloads, as well as convenience methods for interacting with the node over P2P. Individual testcases should subclass this and override the on_* methods if they want to alter message handling behaviour.""" def __init__(self): super().__init__() # Track number of messages of each type received and the most recent # message of each type self.message_count = defaultdict(int) self.last_message = {} # A count of the number of ping messages we've sent to the node self.ping_counter = 1 # The network services received from the peer self.nServices = 0 def peer_connect(self, *args, services=NODE_NETWORK, send_version=True, **kwargs): create_conn = super().peer_connect(*args, **kwargs) if send_version: # Send a version msg vt = msg_version() vt.nServices = services vt.addrTo.ip = self.dstaddr vt.addrTo.port = self.dstport vt.addrFrom.ip = "0.0.0.0" vt.addrFrom.port = 0 # Will be sent soon after connection_made self.on_connection_send_msg = vt return create_conn # Message receiving methods def on_message(self, message): """Receive message and dispatch message to appropriate callback. We keep a count of how many of each message type has been received and the most recent message of each type.""" with mininode_lock: try: command = message.command.decode('ascii') self.message_count[command] += 1 self.last_message[command] = message getattr(self, 'on_' + command)(message) except: print("ERROR delivering {} ({})".format( repr(message), sys.exc_info()[0])) raise # Callback methods. Can be overridden by subclasses in individual test # cases to provide custom message handling behaviour. def on_open(self): pass def on_close(self): pass def on_addr(self, message): pass def on_block(self, message): pass def on_blocktxn(self, message): pass def on_cmpctblock(self, message): pass def on_feefilter(self, message): pass def on_getaddr(self, message): pass def on_getblocks(self, message): pass def on_getblocktxn(self, message): pass def on_getdata(self, message): pass def on_getheaders(self, message): pass def on_headers(self, message): pass def on_mempool(self, message): pass def on_notfound(self, message): pass def on_pong(self, message): pass def on_reject(self, message): pass def on_sendcmpct(self, message): pass def on_sendheaders(self, message): pass def on_tx(self, message): pass def on_inv(self, message): want = msg_getdata() for i in message.inv: if i.type != 0: want.inv.append(i) if len(want.inv): self.send_message(want) def on_ping(self, message): self.send_message(msg_pong(message.nonce)) def on_verack(self, message): self.verack_received = True def on_version(self, message): assert message.nVersion >= MIN_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format( message.nVersion, MIN_VERSION_SUPPORTED) self.send_message(msg_verack()) self.nServices = message.nServices # Connection helper methods def wait_for_disconnect(self, timeout=60): def test_function(): return not self.is_connected wait_until(test_function, timeout=timeout, lock=mininode_lock) # Message receiving helper methods def wait_for_block(self, blockhash, timeout=60): def test_function(): return self.last_message.get( "block") and self.last_message["block"].block.rehash() == blockhash wait_until(test_function, timeout=timeout, lock=mininode_lock) def wait_for_getdata(self, timeout=60): """Waits for a getdata message. Receiving any getdata message will satisfy the predicate. the last_message["getdata"] value must be explicitly cleared before calling this method, or this will return immediately with success. TODO: change this method to take a hash value and only return true if the correct block/tx has been requested.""" def test_function(): return self.last_message.get("getdata") wait_until(test_function, timeout=timeout, lock=mininode_lock) def wait_for_getheaders(self, timeout=60): """Waits for a getheaders message. Receiving any getheaders message will satisfy the predicate. the last_message["getheaders"] value must be explicitly cleared before calling this method, or this will return immediately with success. TODO: change this method to take a hash value and only return true if the correct block header has been requested.""" def test_function(): return self.last_message.get("getheaders") wait_until(test_function, timeout=timeout, lock=mininode_lock) def wait_for_inv(self, expected_inv, timeout=60): """Waits for an INV message and checks that the first inv object in the message was as expected.""" if len(expected_inv) > 1: raise NotImplementedError( "wait_for_inv() will only verify the first inv object") def test_function(): return self.last_message.get("inv") and \ self.last_message["inv"].inv[0].type == expected_inv[0].type and \ self.last_message["inv"].inv[0].hash == expected_inv[0].hash wait_until(test_function, timeout=timeout, lock=mininode_lock) def wait_for_verack(self, timeout=60): def test_function(): return self.message_count["verack"] wait_until(test_function, timeout=timeout, lock=mininode_lock) # Message sending helper functions def send_and_ping(self, message): self.send_message(message) self.sync_with_ping() # Sync up with the node def sync_with_ping(self, timeout=60): self.send_message(msg_ping(nonce=self.ping_counter)) def test_function(): if not self.last_message.get("pong"): return False return self.last_message["pong"].nonce == self.ping_counter wait_until(test_function, timeout=timeout, lock=mininode_lock) self.ping_counter += 1 # One lock for synchronizing all data access between the networking thread (see # NetworkThread below) and the thread running the test logic. For simplicity, # P2PConnection acquires this lock whenever delivering a message to a P2PInterface. # This lock should be acquired in the thread running the test logic to synchronize # access to any data shared with the P2PInterface or P2PConnection. mininode_lock = threading.RLock() class NetworkThread(threading.Thread): network_event_loop = None def __init__(self): super().__init__(name="NetworkThread") # There is only one event loop and no more than one thread must be created assert not self.network_event_loop NetworkThread.network_event_loop = asyncio.new_event_loop() def run(self): """Start the network thread.""" self.network_event_loop.run_forever() def close(self, timeout=10): """Close the connections and network event loop.""" self.network_event_loop.call_soon_threadsafe( self.network_event_loop.stop) wait_until(lambda: not self.network_event_loop.is_running(), timeout=timeout) self.network_event_loop.close() self.join(timeout) class P2PDataStore(P2PInterface): """A P2P data store class. Keeps a block and transaction store and responds correctly to getdata and getheaders requests.""" def __init__(self): super().__init__() # store of blocks. key is block hash, value is a CBlock object self.block_store = {} self.last_block_hash = '' # store of txs. key is txid, value is a CTransaction object self.tx_store = {} self.getdata_requests = [] def on_getdata(self, message): """Check for the tx/block in our stores and if found, reply with an inv message.""" for inv in message.inv: self.getdata_requests.append(inv.hash) if (inv.type & MSG_TYPE_MASK) == MSG_TX and inv.hash in self.tx_store.keys(): self.send_message(msg_tx(self.tx_store[inv.hash])) elif (inv.type & MSG_TYPE_MASK) == MSG_BLOCK and inv.hash in self.block_store.keys(): self.send_message(msg_block(self.block_store[inv.hash])) else: logger.debug( 'getdata message type {} received.'.format(hex(inv.type))) def on_getheaders(self, message): """Search back through our block store for the locator, and reply with a headers message if found.""" locator, hash_stop = message.locator, message.hashstop # Assume that the most recent block added is the tip if not self.block_store: return headers_list = [self.block_store[self.last_block_hash]] maxheaders = 2000 while headers_list[-1].sha256 not in locator.vHave: # Walk back through the block store, adding headers to headers_list # as we go. prev_block_hash = headers_list[-1].hashPrevBlock if prev_block_hash in self.block_store: prev_block_header = CBlockHeader( self.block_store[prev_block_hash]) headers_list.append(prev_block_header) if prev_block_header.sha256 == hash_stop: # if this is the hashstop header, stop here break else: logger.debug('block hash {} not found in block store'.format( hex(prev_block_hash))) break # Truncate the list if there are too many headers headers_list = headers_list[:-maxheaders - 1:-1] response = msg_headers(headers_list) if response is not None: self.send_message(response) def send_blocks_and_test(self, blocks, node, *, success=True, request_block=True, reject_reason=None, expect_disconnect=False, timeout=60): """Send blocks to test node and test whether the tip advances. - add all blocks to our block_store - send a headers message for the final block - the on_getheaders handler will ensure that any getheaders are responded to - if request_block is True: wait for getdata for each of the blocks. The on_getdata handler will ensure that any getdata messages are responded to - if success is True: assert that the node's tip advances to the most recent block - if success is False: assert that the node's tip doesn't advance - if reject_reason is set: assert that the correct reject message is logged""" with mininode_lock: for block in blocks: self.block_store[block.sha256] = block self.last_block_hash = block.sha256 reject_reason = [reject_reason] if reject_reason else [] with node.assert_debug_log(expected_msgs=reject_reason): self.send_message(msg_headers([CBlockHeader(blocks[-1])])) if request_block: wait_until( lambda: blocks[-1].sha256 in self.getdata_requests, timeout=timeout, lock=mininode_lock) if expect_disconnect: self.wait_for_disconnect() else: self.sync_with_ping() if success: wait_until(lambda: node.getbestblockhash() == blocks[-1].hash, timeout=timeout) else: assert node.getbestblockhash() != blocks[-1].hash def send_txs_and_test(self, txs, node, *, success=True, expect_disconnect=False, reject_reason=None): """Send txs to test node and test whether they're accepted to the mempool. - add all txs to our tx_store - send tx messages for all txs - if success is True/False: assert that the txs are/are not accepted to the mempool - if expect_disconnect is True: Skip the sync with ping - if reject_reason is set: assert that the correct reject message is logged.""" with mininode_lock: for tx in txs: self.tx_store[tx.sha256] = tx reject_reason = [reject_reason] if reject_reason else [] with node.assert_debug_log(expected_msgs=reject_reason): for tx in txs: self.send_message(msg_tx(tx)) if expect_disconnect: self.wait_for_disconnect() else: self.sync_with_ping() raw_mempool = node.getrawmempool() if success: # Check that all txs are now in the mempool for tx in txs: assert tx.hash in raw_mempool, "{} not found in mempool".format( tx.hash) else: # Check that none of the txs are now in the mempool for tx in txs: assert tx.hash not in raw_mempool, "{} tx found in mempool".format( tx.hash) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 5d731c68f..49bfe8c4a 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -1,462 +1,459 @@ #!/usr/bin/env python3 # Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Class for bitcoind node under test""" import contextlib import decimal from enum import Enum import errno import http.client import json import logging import os import re import subprocess import sys import tempfile import time from .authproxy import JSONRPCException from .messages import COIN, CTransaction, FromHex from .util import ( append_config, delete_cookie_file, get_rpc_proxy, p2p_port, rpc_url, wait_until, ) -# For Python 3.4 compatibility -JSONDecodeError = getattr(json, "JSONDecodeError", ValueError) - BITCOIND_PROC_WAIT_TIMEOUT = 60 class FailedToStartError(Exception): """Raised when a node fails to start correctly.""" class ErrorMatch(Enum): FULL_TEXT = 1 FULL_REGEX = 2 PARTIAL_REGEX = 3 class TestNode(): """A class for representing a bitcoind node under test. This class contains: - state about the node (whether it's running, etc) - a Python subprocess.Popen object representing the running process - an RPC connection to the node - one or more P2P connections to the node To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" def __init__(self, i, datadir, host, rpc_port, p2p_port, timewait, bitcoind, bitcoin_cli, mocktime, coverage_dir, extra_conf=None, extra_args=None, use_cli=False): self.index = i self.datadir = datadir self.stdout_dir = os.path.join(self.datadir, "stdout") self.stderr_dir = os.path.join(self.datadir, "stderr") self.host = host self.rpc_port = rpc_port self.p2p_port = p2p_port self.name = "testnode-{}".format(i) if timewait: self.rpc_timeout = timewait else: # Wait for up to 60 seconds for the RPC server to respond self.rpc_timeout = 60 self.binary = bitcoind if not os.path.isfile(self.binary): raise FileNotFoundError( "Binary '{}' could not be found.\nTry setting it manually:\n\tBITCOIND= {}".format(self.binary, sys.argv[0])) self.coverage_dir = coverage_dir if extra_conf != None: append_config(datadir, extra_conf) # Most callers will just need to add extra args to the default list # below. # For those callers that need more flexibility, they can access the # default args using the provided facilities. # Note that common args are set in the config file (see # initialize_datadir) self.extra_args = extra_args self.default_args = ["-datadir=" + self.datadir, "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(mocktime), "-uacomment=" + self.name] if not os.path.isfile(bitcoin_cli): raise FileNotFoundError( "Binary '{}' could not be found.\nTry setting it manually:\n\tBITCOINCLI= {}".format(bitcoin_cli, sys.argv[0])) self.cli = TestNodeCLI(bitcoin_cli, self.datadir) self.use_cli = use_cli self.running = False self.process = None self.rpc_connected = False self.rpc = None self.url = None self.relay_fee_cache = None self.log = logging.getLogger('TestFramework.node{}'.format(i)) # Whether to kill the node when this object goes away self.cleanup_on_exit = True self.p2ps = [] def _node_msg(self, msg: str) -> str: """Return a modified msg that identifies this node by its index as a debugging aid.""" return "[node {}] {}".format(self.index, msg) def _raise_assertion_error(self, msg: str): """Raise an AssertionError with msg modified to identify this node.""" raise AssertionError(self._node_msg(msg)) def __del__(self): # Ensure that we don't leave any bitcoind processes lying around after # the test ends if self.process and self.cleanup_on_exit: # Should only happen on test failure # Avoid using logger, as that may have already been shutdown when # this destructor is called. print(self._node_msg("Cleaning up leftover process")) self.process.kill() def __getattr__(self, name): """Dispatches any unrecognised messages to the RPC connection or a CLI instance.""" if self.use_cli: return getattr(self.cli, name) else: assert self.rpc is not None, self._node_msg( "Error: RPC not initialized") assert self.rpc_connected, self._node_msg( "Error: No RPC connection") return getattr(self.rpc, name) def clear_default_args(self): self.default_args.clear() def extend_default_args(self, args): self.default_args.extend(args) def remove_default_args(self, args): for rm_arg in args: # Remove all occurrences of rm_arg in self.default_args: # - if the arg is a flag (-flag), then the names must match # - if the arg is a value (-key=value) then the name must starts # with "-key=" (the '"' char is to avoid removing "-key_suffix" # arg is "-key" is the argument to remove). self.default_args = [def_arg for def_arg in self.default_args if rm_arg != def_arg and not def_arg.startswith(rm_arg + '=')] def start(self, extra_args=None, stdout=None, stderr=None, *args, **kwargs): """Start the node.""" if extra_args is None: extra_args = self.extra_args # Add a new stdout and stderr file each time bitcoind is started if stderr is None: stderr = tempfile.NamedTemporaryFile( dir=self.stderr_dir, delete=False) if stdout is None: stdout = tempfile.NamedTemporaryFile( dir=self.stdout_dir, delete=False) self.stderr = stderr self.stdout = stdout # Delete any existing cookie file -- if such a file exists (eg due to # unclean shutdown), it will get overwritten anyway by bitcoind, and # potentially interfere with our attempt to authenticate delete_cookie_file(self.datadir) # add environment variable LIBC_FATAL_STDERR_=1 so that libc errors are written to stderr and not the terminal subp_env = dict(os.environ, LIBC_FATAL_STDERR_="1") self.process = subprocess.Popen( [self.binary] + self.default_args + extra_args, env=subp_env, stdout=stdout, stderr=stderr, *args, **kwargs) self.running = True self.log.debug("bitcoind started, waiting for RPC to come up") def wait_for_rpc_connection(self): """Sets up an RPC connection to the bitcoind process. Returns False if unable to connect.""" # Poll at a rate of four times per second poll_per_s = 4 for _ in range(poll_per_s * self.rpc_timeout): if self.process.poll() is not None: raise FailedToStartError(self._node_msg( 'bitcoind exited with status {} during initialization'.format(self.process.returncode))) try: self.rpc = get_rpc_proxy(rpc_url(self.datadir, self.host, self.rpc_port), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir) self.rpc.getblockcount() # If the call to getblockcount() succeeds then the RPC connection is up self.rpc_connected = True self.url = self.rpc.url self.log.debug("RPC successfully started") return except IOError as e: if e.errno != errno.ECONNREFUSED: # Port not yet open? raise # unknown IO error except JSONRPCException as e: # Initialization phase if e.error['code'] != -28: # RPC in warmup? raise # unknown JSON RPC exception except ValueError as e: # cookie file not found and no rpcuser or rpcassword. bitcoind still starting if "No RPC credentials" not in str(e): raise time.sleep(1.0 / poll_per_s) self._raise_assertion_error("Unable to connect to bitcoind") def get_wallet_rpc(self, wallet_name): if self.use_cli: return self.cli("-rpcwallet={}".format(wallet_name)) else: assert self.rpc is not None, self._node_msg( "Error: RPC not initialized") assert self.rpc_connected, self._node_msg( "Error: RPC not connected") wallet_path = "wallet/{}".format(wallet_name) return self.rpc / wallet_path def stop_node(self, expected_stderr=''): """Stop the node.""" if not self.running: return self.log.debug("Stopping node") try: self.stop() except http.client.CannotSendRequest: self.log.exception("Unable to stop node.") # Check that stderr is as expected self.stderr.seek(0) stderr = self.stderr.read().decode('utf-8').strip() if stderr != expected_stderr: raise AssertionError( "Unexpected stderr {} != {}".format(stderr, expected_stderr)) del self.p2ps[:] def is_node_stopped(self): """Checks whether the node has stopped. Returns True if the node has stopped. False otherwise. This method is responsible for freeing resources (self.process).""" if not self.running: return True return_code = self.process.poll() if return_code is None: return False # process has stopped. Assert that it didn't return an error code. assert return_code == 0, self._node_msg( "Node returned non-zero exit code ({}) when stopping".format(return_code)) self.running = False self.process = None self.rpc_connected = False self.rpc = None self.log.debug("Node stopped") return True def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT): wait_until(self.is_node_stopped, timeout=timeout) @contextlib.contextmanager def assert_debug_log(self, expected_msgs): debug_log = os.path.join(self.datadir, 'regtest', 'debug.log') with open(debug_log, encoding='utf-8') as dl: dl.seek(0, 2) prev_size = dl.tell() try: yield finally: with open(debug_log, encoding='utf-8') as dl: dl.seek(prev_size) log = dl.read() print_log = " - " + "\n - ".join(log.splitlines()) for expected_msg in expected_msgs: if re.search(re.escape(expected_msg), log, flags=re.MULTILINE) is None: self._raise_assertion_error( 'Expected message "{}" does not partially match log:\n\n{}\n\n'.format(expected_msg, print_log)) def assert_start_raises_init_error(self, extra_args=None, expected_msg=None, match=ErrorMatch.FULL_TEXT, *args, **kwargs): """Attempt to start the node and expect it to raise an error. extra_args: extra arguments to pass through to bitcoind expected_msg: regex that stderr should match when bitcoind fails Will throw if bitcoind starts without an error. Will throw if an expected_msg is provided and it does not match bitcoind's stdout.""" with tempfile.NamedTemporaryFile(dir=self.stderr_dir, delete=False) as log_stderr, \ tempfile.NamedTemporaryFile(dir=self.stdout_dir, delete=False) as log_stdout: try: self.start(extra_args, stdout=log_stdout, stderr=log_stderr, *args, **kwargs) self.wait_for_rpc_connection() self.stop_node() self.wait_until_stopped() except FailedToStartError as e: self.log.debug('bitcoind failed to start: {}'.format(e)) self.running = False self.process = None # Check stderr for expected message if expected_msg is not None: log_stderr.seek(0) stderr = log_stderr.read().decode('utf-8').strip() if match == ErrorMatch.PARTIAL_REGEX: if re.search(expected_msg, stderr, flags=re.MULTILINE) is None: self._raise_assertion_error( 'Expected message "{}" does not partially match stderr:\n"{}"'.format(expected_msg, stderr)) elif match == ErrorMatch.FULL_REGEX: if re.fullmatch(expected_msg, stderr) is None: self._raise_assertion_error( 'Expected message "{}" does not fully match stderr:\n"{}"'.format(expected_msg, stderr)) elif match == ErrorMatch.FULL_TEXT: if expected_msg != stderr: self._raise_assertion_error( 'Expected message "{}" does not fully match stderr:\n"{}"'.format(expected_msg, stderr)) else: if expected_msg is None: assert_msg = "bitcoind should have exited with an error" else: assert_msg = "bitcoind should have exited with expected error " + expected_msg self._raise_assertion_error(assert_msg) def node_encrypt_wallet(self, passphrase): """"Encrypts the wallet. This causes bitcoind to shutdown, so this method takes care of cleaning up resources.""" self.encryptwallet(passphrase) self.wait_until_stopped() def relay_fee(self, cached=True): if not self.relay_fee_cache or not cached: self.relay_fee_cache = self.getnetworkinfo()["relayfee"] return self.relay_fee_cache def calculate_fee(self, tx): # Relay fee is in satoshis per KB. Thus the 1000, and the COIN added # to get back to an amount of satoshis. billable_size_estimate = tx.billable_size() # Add some padding for signatures # NOTE: Fees must be calculated before signatures are added, # so they will never be included in the billable_size above. billable_size_estimate += len(tx.vin) * 81 return int(self.relay_fee() / 1000 * billable_size_estimate * COIN) def calculate_fee_from_txid(self, txid): ctx = FromHex(CTransaction(), self.getrawtransaction(txid)) return self.calculate_fee(ctx) def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, **kwargs): """Add a p2p connection to the node. This method adds the p2p connection to the self.p2ps list and also returns the connection to the caller.""" if 'dstport' not in kwargs: kwargs['dstport'] = p2p_port(self.index) if 'dstaddr' not in kwargs: kwargs['dstaddr'] = '127.0.0.1' p2p_conn.peer_connect(**kwargs)() self.p2ps.append(p2p_conn) if wait_for_verack: p2p_conn.wait_for_verack() return p2p_conn @property def p2p(self): """Return the first p2p connection Convenience property - most tests only use a single p2p connection to each node, so this saves having to write node.p2ps[0] many times.""" assert self.p2ps, self._node_msg("No p2p connection") return self.p2ps[0] def disconnect_p2ps(self): """Close all p2p connections to the node.""" for p in self.p2ps: p.peer_disconnect() del self.p2ps[:] class TestNodeCLIAttr: def __init__(self, cli, command): self.cli = cli self.command = command def __call__(self, *args, **kwargs): return self.cli.send_cli(self.command, *args, **kwargs) def get_request(self, *args, **kwargs): return lambda: self(*args, **kwargs) class TestNodeCLI(): """Interface to bitcoin-cli for an individual node""" def __init__(self, binary, datadir): self.options = [] self.binary = binary self.datadir = datadir self.input = None self.log = logging.getLogger('TestFramework.bitcoincli') def __call__(self, *options, input=None): # TestNodeCLI is callable with bitcoin-cli command-line options cli = TestNodeCLI(self.binary, self.datadir) cli.options = [str(o) for o in options] cli.input = input return cli def __getattr__(self, command): return TestNodeCLIAttr(self, command) def batch(self, requests): results = [] for request in requests: try: results.append(dict(result=request())) except JSONRPCException as e: results.append(dict(error=e)) return results def send_cli(self, command=None, *args, **kwargs): """Run bitcoin-cli command. Deserializes returned string as python object.""" pos_args = [str(arg) for arg in args] named_args = [str(key) + "=" + str(value) for (key, value) in kwargs.items()] assert not ( pos_args and named_args), "Cannot use positional arguments and named arguments in the same bitcoin-cli call" p_args = [self.binary, "-datadir=" + self.datadir] + self.options if named_args: p_args += ["-named"] if command is not None: p_args += [command] p_args += pos_args + named_args self.log.debug("Running bitcoin-cli command: {}".format(command)) process = subprocess.Popen(p_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) cli_stdout, cli_stderr = process.communicate(input=self.input) returncode = process.poll() if returncode: match = re.match( r'error code: ([-0-9]+)\nerror message:\n(.*)', cli_stderr) if match: code, message = match.groups() raise JSONRPCException(dict(code=int(code), message=message)) # Ignore cli_stdout, raise with cli_stderr raise subprocess.CalledProcessError( returncode, self.binary, output=cli_stderr) try: return json.loads(cli_stdout, parse_float=decimal.Decimal) - except JSONDecodeError: + except json.JSONDecodeError: return cli_stdout.rstrip("\n") diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 5195a3cee..96d480a64 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -1,648 +1,644 @@ #!/usr/bin/env python3 # Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helpful routines for regression testing.""" from base64 import b64encode from binascii import unhexlify from decimal import Decimal, ROUND_DOWN import hashlib import inspect import json import logging import os import random import re from subprocess import CalledProcessError import time from . import coverage from .authproxy import AuthServiceProxy, JSONRPCException logger = logging.getLogger("TestFramework.utils") # Assert functions ################## def assert_fee_amount(fee, tx_size, fee_per_kB, wiggleroom=2): """ Assert the fee was in range wiggleroom defines an amount that the test expects the wallet to be off by when estimating fees. This can be due to the dummy signature that is added during fee calculation, or due to the wallet funding transactions using the ceiling of the calculated fee. """ target_fee = round(tx_size * fee_per_kB / 1000, 8) if fee < (tx_size - wiggleroom) * fee_per_kB / 1000: raise AssertionError( "Fee of {} BCH too low! (Should be {} BCH)".format(str(fee), str(target_fee))) if fee > (tx_size + wiggleroom) * fee_per_kB / 1000: raise AssertionError( "Fee of {} BCH too high! (Should be {} BCH)".format(str(fee), str(target_fee))) def assert_equal(thing1, thing2, *args): if thing1 != thing2 or any(thing1 != arg for arg in args): raise AssertionError("not({})".format(" == ".join(str(arg) for arg in (thing1, thing2) + args))) def assert_greater_than(thing1, thing2): if thing1 <= thing2: raise AssertionError("{} <= {}".format(str(thing1), str(thing2))) def assert_greater_than_or_equal(thing1, thing2): if thing1 < thing2: raise AssertionError("{} < {}".format(str(thing1), str(thing2))) def assert_raises(exc, fun, *args, **kwds): assert_raises_message(exc, None, fun, *args, **kwds) def assert_raises_message(exc, message, fun, *args, **kwds): try: fun(*args, **kwds) except JSONRPCException: raise AssertionError( "Use assert_raises_rpc_error() to test RPC failures") except exc as e: if message is not None and message not in e.error['message']: raise AssertionError( "Expected substring not found:" + e.error['message']) except Exception as e: raise AssertionError( "Unexpected exception raised: " + type(e).__name__) else: raise AssertionError("No exception raised") def assert_raises_process_error(returncode, output, fun, *args, **kwds): """Execute a process and asserts the process return code and output. Calls function `fun` with arguments `args` and `kwds`. Catches a CalledProcessError and verifies that the return code and output are as expected. Throws AssertionError if no CalledProcessError was raised or if the return code and output are not as expected. Args: returncode (int): the process return code. output (string): [a substring of] the process output. fun (function): the function to call. This should execute a process. args*: positional arguments for the function. kwds**: named arguments for the function. """ try: fun(*args, **kwds) except CalledProcessError as e: if returncode != e.returncode: raise AssertionError( "Unexpected returncode {}".format(e.returncode)) if output not in e.output: raise AssertionError("Expected substring not found:" + e.output) else: raise AssertionError("No exception raised") def assert_raises_rpc_error(code, message, fun, *args, **kwds): """Run an RPC and verify that a specific JSONRPC exception code and message is raised. Calls function `fun` with arguments `args` and `kwds`. Catches a JSONRPCException and verifies that the error code and message are as expected. Throws AssertionError if no JSONRPCException was raised or if the error code/message are not as expected. Args: code (int), optional: the error code returned by the RPC call (defined in src/rpc/protocol.h). Set to None if checking the error code is not required. message (string), optional: [a substring of] the error string returned by the RPC call. Set to None if checking the error string is not required. fun (function): the function to call. This should be the name of an RPC. args*: positional arguments for the function. kwds**: named arguments for the function. """ assert try_rpc(code, message, fun, *args, **kwds), "No exception raised" def try_rpc(code, message, fun, *args, **kwds): """Tries to run an rpc command. Test against error code and message if the rpc fails. Returns whether a JSONRPCException was raised.""" try: fun(*args, **kwds) except JSONRPCException as e: # JSONRPCException was thrown as expected. Check the code and message values are correct. if (code is not None) and (code != e.error["code"]): raise AssertionError( "Unexpected JSONRPC error code {}".format(e.error["code"])) if (message is not None) and (message not in e.error['message']): raise AssertionError( "Expected substring not found:" + e.error['message']) return True except Exception as e: raise AssertionError( "Unexpected exception raised: " + type(e).__name__) else: return False def assert_is_hex_string(string): try: int(string, 16) except Exception as e: raise AssertionError( "Couldn't interpret {!r} as hexadecimal; raised: {}".format(string, e)) def assert_is_hash_string(string, length=64): if not isinstance(string, str): raise AssertionError( "Expected a string, got type {!r}".format(type(string))) elif length and len(string) != length: raise AssertionError( "String of length {} expected; got {}".format(length, len(string))) elif not re.match('[abcdef0-9]+$', string): raise AssertionError( "String {!r} contains invalid characters for a hash.".format(string)) def assert_array_result(object_array, to_match, expected, should_not_find=False): """ Pass in array of JSON objects, a dictionary with key/value pairs to match against, and another dictionary with expected key/value pairs. If the should_not_find flag is true, to_match should not be found in object_array """ if should_not_find: assert_equal(expected, {}) num_matched = 0 for item in object_array: all_match = True for key, value in to_match.items(): if item[key] != value: all_match = False if not all_match: continue elif should_not_find: num_matched = num_matched + 1 for key, value in expected.items(): if item[key] != value: raise AssertionError("{} : expected {}={}".format( str(item), str(key), str(value))) num_matched = num_matched + 1 if num_matched == 0 and not should_not_find: raise AssertionError("No objects matched {}".format(str(to_match))) if num_matched > 0 and should_not_find: raise AssertionError("Objects were found {}".format(str(to_match))) # Utility functions ################### def check_json_precision(): """Make sure json library being used does not lose precision converting BCH values""" n = Decimal("20000000.00000003") satoshis = int(json.loads(json.dumps(float(n))) * 1.0e8) if satoshis != 2000000000000003: raise RuntimeError("JSON encode/decode loses precision") def count_bytes(hex_string): return len(bytearray.fromhex(hex_string)) -def b_2_x(byte_str): - return byte_str.hex() - - def hash256(byte_str): sha256 = hashlib.sha256() sha256.update(byte_str) sha256d = hashlib.sha256() sha256d.update(sha256.digest()) return sha256d.digest()[::-1] def hex_str_to_bytes(hex_str): return unhexlify(hex_str.encode('ascii')) def str_to_b64str(string): return b64encode(string.encode('utf-8')).decode('ascii') def satoshi_round(amount): return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None): if attempts == float('inf') and timeout == float('inf'): timeout = 60 attempt = 0 time_end = time.time() + timeout while attempt < attempts and time.time() < time_end: if lock: with lock: if predicate(): return else: if predicate(): return attempt += 1 time.sleep(0.05) # Print the cause of the timeout predicate_source = inspect.getsourcelines(predicate) logger.error("wait_until() failed. Predicate: {}".format(predicate_source)) if attempt >= attempts: raise AssertionError("Predicate {} not true after {} attempts".format( predicate_source, attempts)) elif time.time() >= time_end: raise AssertionError( "Predicate {} not true after {} seconds".format(predicate_source, timeout)) raise RuntimeError('Unreachable') # RPC/P2P connection constants and functions ############################################ # The maximum number of nodes a single test can spawn MAX_NODES = 8 # Don't assign rpc or p2p ports lower than this PORT_MIN = 11000 # The number of ports to "reserve" for p2p and rpc, each PORT_RANGE = 5000 class PortSeed: # Must be initialized with a unique integer for each process n = None def get_rpc_proxy(url, node_number, timeout=None, coveragedir=None): """ Args: url (str): URL of the RPC server to call node_number (int): the node number (or id) that this calls to Kwargs: timeout (int): HTTP timeout in seconds Returns: AuthServiceProxy. convenience object for making RPC calls. """ proxy_kwargs = {} if timeout is not None: proxy_kwargs['timeout'] = timeout proxy = AuthServiceProxy(url, **proxy_kwargs) proxy.url = url # store URL on proxy for info coverage_logfile = coverage.get_filename( coveragedir, node_number) if coveragedir else None return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile) def p2p_port(n): assert n <= MAX_NODES return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) def rpc_port(n): return PORT_MIN + PORT_RANGE + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) def rpc_url(datadir, host, port): rpc_u, rpc_p = get_auth_cookie(datadir) if host == None: host = '127.0.0.1' return "http://{}:{}@{}:{}".format(rpc_u, rpc_p, host, int(port)) # Node functions ################ def initialize_datadir(dirname, n): datadir = get_datadir_path(dirname, n) if not os.path.isdir(datadir): os.makedirs(datadir) with open(os.path.join(datadir, "bitcoin.conf"), 'w', encoding='utf8') as f: f.write("regtest=1\n") f.write("[regtest]\n") f.write("port=" + str(p2p_port(n)) + "\n") f.write("rpcport=" + str(rpc_port(n)) + "\n") f.write("server=1\n") f.write("keypool=1\n") f.write("discover=0\n") f.write("listenonion=0\n") f.write("usecashaddr=1\n") os.makedirs(os.path.join(datadir, 'stderr'), exist_ok=True) os.makedirs(os.path.join(datadir, 'stdout'), exist_ok=True) return datadir def get_datadir_path(dirname, n): return os.path.join(dirname, "node" + str(n)) def append_config(datadir, options): with open(os.path.join(datadir, "bitcoin.conf"), 'a', encoding='utf8') as f: for option in options: f.write(option + "\n") def get_auth_cookie(datadir): user = None password = None if os.path.isfile(os.path.join(datadir, "bitcoin.conf")): with open(os.path.join(datadir, "bitcoin.conf"), 'r', encoding='utf8') as f: for line in f: if line.startswith("rpcuser="): assert user is None # Ensure that there is only one rpcuser line user = line.split("=")[1].strip("\n") if line.startswith("rpcpassword="): assert password is None # Ensure that there is only one rpcpassword line password = line.split("=")[1].strip("\n") if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")): with open(os.path.join(datadir, "regtest", ".cookie"), 'r', encoding="ascii") as f: userpass = f.read() split_userpass = userpass.split(':') user = split_userpass[0] password = split_userpass[1] if user is None or password is None: raise ValueError("No RPC credentials") return user, password # If a cookie file exists in the given datadir, delete it. def delete_cookie_file(datadir): if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")): logger.debug("Deleting leftover cookie file") os.remove(os.path.join(datadir, "regtest", ".cookie")) def set_node_times(nodes, t): for node in nodes: node.setmocktime(t) def disconnect_nodes(from_node, to_node): for peer_id in [peer['id'] for peer in from_node.getpeerinfo() if to_node.name in peer['subver']]: try: from_node.disconnectnode(nodeid=peer_id) except JSONRPCException as e: # If this node is disconnected between calculating the peer id # and issuing the disconnect, don't worry about it. # This avoids a race condition if we're mass-disconnecting peers. if e.error['code'] != -29: # RPC_CLIENT_NODE_NOT_CONNECTED raise # wait to disconnect wait_until(lambda: [peer['id'] for peer in from_node.getpeerinfo( ) if to_node.name in peer['subver']] == [], timeout=5) def connect_nodes(from_node, to_node): host = to_node.host if host == None: host = '127.0.0.1' ip_port = host + ':' + str(to_node.p2p_port) from_node.addnode(ip_port, "onetry") # poll until version handshake complete to avoid race conditions # with transaction relaying wait_until(lambda: all(peer['version'] != 0 for peer in from_node.getpeerinfo())) def connect_nodes_bi(a, b): connect_nodes(a, b) connect_nodes(b, a) def sync_blocks(rpc_connections, *, wait=1, timeout=60): """ Wait until everybody has the same tip. sync_blocks needs to be called with an rpc_connections set that has least one node already synced to the latest, stable tip, otherwise there's a chance it might return before all nodes are stably synced. """ stop_time = time.time() + timeout while time.time() <= stop_time: best_hash = [x.getbestblockhash() for x in rpc_connections] if best_hash.count(best_hash[0]) == len(rpc_connections): return time.sleep(wait) raise AssertionError("Block sync timed out:{}".format( "".join("\n {!r}".format(b) for b in best_hash))) def sync_mempools(rpc_connections, *, wait=1, timeout=60, flush_scheduler=True): """ Wait until everybody has the same transactions in their memory pools """ stop_time = time.time() + timeout while time.time() <= stop_time: pool = [set(r.getrawmempool()) for r in rpc_connections] if pool.count(pool[0]) == len(rpc_connections): if flush_scheduler: for r in rpc_connections: r.syncwithvalidationinterfacequeue() return time.sleep(wait) raise AssertionError("Mempool sync timed out:{}".format( "".join("\n {!r}".format(m) for m in pool))) # Transaction/Block functions ############################# def find_output(node, txid, amount): """ Return index to output of txid with value amount Raises exception if there is none. """ txdata = node.getrawtransaction(txid, 1) for i in range(len(txdata["vout"])): if txdata["vout"][i]["value"] == amount: return i raise RuntimeError("find_output txid {} : {} not found".format( txid, str(amount))) def gather_inputs(from_node, amount_needed, confirmations_required=1): """ Return a random set of unspent txouts that are enough to pay amount_needed """ assert confirmations_required >= 0 utxo = from_node.listunspent(confirmations_required) random.shuffle(utxo) inputs = [] total_in = Decimal("0.00000000") while total_in < amount_needed and len(utxo) > 0: t = utxo.pop() total_in += t["amount"] inputs.append( {"txid": t["txid"], "vout": t["vout"], "address": t["address"]}) if total_in < amount_needed: raise RuntimeError("Insufficient funds: need {}, have {}".format( amount_needed, total_in)) return (total_in, inputs) def make_change(from_node, amount_in, amount_out, fee): """ Create change output(s), return them """ outputs = {} amount = amount_out + fee change = amount_in - amount if change > amount * 2: # Create an extra change output to break up big inputs change_address = from_node.getnewaddress() # Split change in two, being careful of rounding: outputs[change_address] = Decimal( change / 2).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) change = amount_in - amount - outputs[change_address] if change > 0: outputs[from_node.getnewaddress()] = change return outputs def send_zeropri_transaction(from_node, to_node, amount, fee): """ Create&broadcast a zero-priority transaction. Returns (txid, hex-encoded-txdata) Ensures transaction is zero-priority by first creating a send-to-self, then using its output """ # Create a send-to-self with confirmed inputs: self_address = from_node.getnewaddress() (total_in, inputs) = gather_inputs(from_node, amount + fee * 2) outputs = make_change(from_node, total_in, amount + fee, fee) outputs[self_address] = float(amount + fee) self_rawtx = from_node.createrawtransaction(inputs, outputs) self_signresult = from_node.signrawtransactionwithwallet(self_rawtx) self_txid = from_node.sendrawtransaction(self_signresult["hex"], True) vout = find_output(from_node, self_txid, amount + fee) # Now immediately spend the output to create a 1-input, 1-output # zero-priority transaction: inputs = [{"txid": self_txid, "vout": vout}] outputs = {to_node.getnewaddress(): float(amount)} rawtx = from_node.createrawtransaction(inputs, outputs) signresult = from_node.signrawtransactionwithwallet(rawtx) txid = from_node.sendrawtransaction(signresult["hex"], True) return (txid, signresult["hex"]) def random_zeropri_transaction(nodes, amount, min_fee, fee_increment, fee_variants): """ Create a random zero-priority transaction. Returns (txid, hex-encoded-transaction-data, fee) """ from_node = random.choice(nodes) to_node = random.choice(nodes) fee = min_fee + fee_increment * random.randint(0, fee_variants) (txid, txhex) = send_zeropri_transaction(from_node, to_node, amount, fee) return (txid, txhex, fee) def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): """ Create a random transaction. Returns (txid, hex-encoded-transaction-data, fee) """ from_node = random.choice(nodes) to_node = random.choice(nodes) fee = min_fee + fee_increment * random.randint(0, fee_variants) (total_in, inputs) = gather_inputs(from_node, amount + fee) outputs = make_change(from_node, total_in, amount, fee) outputs[to_node.getnewaddress()] = float(amount) rawtx = from_node.createrawtransaction(inputs, outputs) signresult = from_node.signrawtransactionwithwallet(rawtx) txid = from_node.sendrawtransaction(signresult["hex"], True) return (txid, signresult["hex"], fee) # Create large OP_RETURN txouts that can be appended to a transaction # to make it large (helper for constructing large transactions). def gen_return_txouts(): # Some pre-processing to create a bunch of OP_RETURN txouts to insert into transactions we create # So we have big transactions (and therefore can't fit very many into each block) # create one script_pubkey script_pubkey = "6a4d0200" # OP_RETURN OP_PUSH2 512 bytes for i in range(512): script_pubkey = script_pubkey + "01" # concatenate 128 txouts of above script_pubkey which we'll insert before # the txout for change txouts = "81" for k in range(128): # add txout value txouts = txouts + "0000000000000000" # add length of script_pubkey txouts = txouts + "fd0402" # add script_pubkey txouts = txouts + script_pubkey return txouts def create_tx(node, coinbase, to_address, amount): inputs = [{"txid": coinbase, "vout": 0}] outputs = {to_address: amount} rawtx = node.createrawtransaction(inputs, outputs) signresult = node.signrawtransactionwithwallet(rawtx) assert_equal(signresult["complete"], True) return signresult["hex"] # Create a spend of each passed-in utxo, splicing in "txouts" to each raw # transaction to make it large. See gen_return_txouts() above. def create_lots_of_big_transactions(node, txouts, utxos, num, fee): addr = node.getnewaddress() txids = [] for _ in range(num): t = utxos.pop() inputs = [{"txid": t["txid"], "vout": t["vout"]}] outputs = {} change = t['amount'] - fee outputs[addr] = satoshi_round(change) rawtx = node.createrawtransaction(inputs, outputs) newtx = rawtx[0:92] newtx = newtx + txouts newtx = newtx + rawtx[94:] signresult = node.signrawtransactionwithwallet( newtx, None, "NONE|FORKID") txid = node.sendrawtransaction(signresult["hex"], True) txids.append(txid) return txids def find_vout_for_address(node, txid, addr): """ Locate the vout index of the given transaction sending to the given address. Raises runtime error exception if not found. """ tx = node.getrawtransaction(txid, True) for i in range(len(tx["vout"])): if any([addr == a for a in tx["vout"][i]["scriptPubKey"]["addresses"]]): return i raise RuntimeError( "Vout not found for address: txid={}, addr={}".format(txid, addr))