diff --git a/depends/Makefile b/depends/Makefile index 6916cadab..302cb2fcb 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -1,234 +1,241 @@ .NOTPARALLEL : # Pattern rule to print variables, e.g. make print-build_arch print-%: @echo $* = $($*) SOURCES_PATH ?= $(BASEDIR)/sources WORK_PATH = $(BASEDIR)/work BASE_CACHE ?= $(BASEDIR)/built SDK_PATH ?= $(BASEDIR)/SDKs NO_QT ?= NO_PROTOBUF ?= NO_QR ?= RAPIDCHECK ?= NO_WALLET ?= NO_ZMQ ?= NO_UPNP ?= +NO_JEMALLOC ?= FALLBACK_DOWNLOAD_PATH ?= https://download.bitcoinabc.org/depends-sources BUILD = $(shell ./config.guess) HOST ?= $(BUILD) PATCHES_PATH = $(BASEDIR)/patches BASEDIR = $(CURDIR) HASH_LENGTH:=11 DOWNLOAD_CONNECT_TIMEOUT:=30 DOWNLOAD_RETRIES:=3 HOST_ID_SALT ?= salt BUILD_ID_SALT ?= salt JOBS ?= $(shell echo $$(($(shell nproc 2> /dev/null || sysctl -n hw.ncpu 2> /dev/null || echo 0) + 1))) host:=$(BUILD) ifneq ($(HOST),) host:=$(HOST) host_toolchain:=$(HOST)- endif ifneq ($(DEBUG),) release_type=debug else release_type=release endif base_build_dir=$(WORK_PATH)/build base_staging_dir=$(WORK_PATH)/staging base_download_dir=$(WORK_PATH)/download canonical_host:=$(shell ./config.sub $(HOST)) build:=$(shell ./config.sub $(BUILD)) build_arch =$(firstword $(subst -, ,$(build))) build_vendor=$(word 2,$(subst -, ,$(build))) full_build_os:=$(subst $(build_arch)-$(build_vendor)-,,$(build)) build_os:=$(findstring linux,$(full_build_os)) build_os+=$(findstring darwin,$(full_build_os)) build_os:=$(strip $(build_os)) ifeq ($(build_os),) build_os=$(full_build_os) endif host_arch=$(firstword $(subst -, ,$(canonical_host))) host_vendor=$(word 2,$(subst -, ,$(canonical_host))) full_host_os:=$(subst $(host_arch)-$(host_vendor)-,,$(canonical_host)) host_os:=$(findstring linux,$(full_host_os)) host_os+=$(findstring darwin,$(full_host_os)) host_os+=$(findstring mingw32,$(full_host_os)) host_os:=$(strip $(host_os)) ifeq ($(host_os),) host_os=$(full_host_os) endif $(host_arch)_$(host_os)_prefix=$(BASEDIR)/$(host) $(host_arch)_$(host_os)_host=$(host) host_prefix=$($(host_arch)_$(host_os)_prefix) build_prefix=$(host_prefix)/native build_host=$(build) AT_$(V):= AT_:=@ AT:=$(AT_$(V)) all: install include hosts/$(host_os).mk include hosts/default.mk include builders/$(build_os).mk include builders/default.mk include packages/packages.mk build_id_string:=$(BUILD_ID_SALT) build_id_string+=$(shell $(build_CC) --version 2>/dev/null) build_id_string+=$(shell $(build_AR) --version 2>/dev/null) build_id_string+=$(shell $(build_CXX) --version 2>/dev/null) build_id_string+=$(shell $(build_RANLIB) --version 2>/dev/null) build_id_string+=$(shell $(build_STRIP) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string:=$(HOST_ID_SALT) $(host_arch)_$(host_os)_id_string+=$(shell $(host_CC) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_AR) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) qrencode_packages_$(NO_QR) = $(qrencode_packages) qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) $(qrencode_packages_) wallet_packages_$(NO_WALLET) = $(wallet_packages) upnp_packages_$(NO_UPNP) = $(upnp_packages) zmq_packages_$(NO_ZMQ) = $(zmq_packages) protobuf_packages_$(NO_PROTOBUF) = $(protobuf_packages) rapidcheck_packages_$(RAPIDCHECK) = $(rapidcheck_packages) +jemalloc_packages_$(NO_JEMALLOC) = $(jemalloc_packages) + packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) ifneq ($(protobuf_packages_),) native_packages += $(protobuf_native_packages) packages += $(protobuf_packages) endif ifneq ($(zmq_packages_),) packages += $(zmq_packages) endif ifeq ($(rapidcheck_packages_),) packages += $(rapidcheck_packages) endif +ifneq ($(jemalloc_packages_),) +packages += $(jemalloc_packages) +endif + all_packages = $(packages) $(native_packages) meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) include funcs.mk toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)rm -rf $(@D) $(AT)mkdir -p $(@D) $(AT)echo copying packages: $^ $(AT)echo to: $(@D) $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) $(AT)touch $@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id) $(AT)@mkdir -p $(@D) $(AT)sed -e 's|@HOST@|$(host)|' \ -e 's|@CC@|$(toolchain_path)$(host_CC)|' \ -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \ -e 's|@AR@|$(toolchain_path)$(host_AR)|' \ -e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \ -e 's|@NM@|$(toolchain_path)$(host_NM)|' \ -e 's|@STRIP@|$(toolchain_path)$(host_STRIP)|' \ -e 's|@build_os@|$(build_os)|' \ -e 's|@host_os@|$(host_os)|' \ -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \ -e 's|@CXXFLAGS@|$(strip $(host_CXXFLAGS) $(host_$(release_type)_CXXFLAGS))|' \ -e 's|@CPPFLAGS@|$(strip $(host_CPPFLAGS) $(host_$(release_type)_CPPFLAGS))|' \ -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \ -e 's|@no_qt@|$(NO_QT)|' \ -e 's|@no_qr@|$(NO_QR)|' \ -e 's|@no_zmq@|$(NO_ZMQ)|' \ -e 's|@no_bip70@|$(NO_PROTOBUF)|' \ -e 's|@no_wallet@|$(NO_WALLET)|' \ -e 's|@no_upnp@|$(NO_UPNP)|' \ -e 's|@debug@|$(DEBUG)|' \ $< > $@ $(AT)touch $@ define check_or_remove_cached mkdir -p $(BASE_CACHE)/$(host)/$(package) && cd $(BASE_CACHE)/$(host)/$(package); \ $(build_SHA256SUM) -c $($(package)_cached_checksum) >/dev/null 2>/dev/null || \ ( rm -f $($(package)_cached_checksum); \ if test -f "$($(package)_cached)"; then echo "Checksum mismatch for $(package). Forcing rebuild.."; rm -f $($(package)_cached_checksum) $($(package)_cached); fi ) endef define check_or_remove_sources mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \ test -f $($(package)_fetched) && ( $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \ ( echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; \ rm -f $($(package)_all_sources) $($(1)_fetched))) || true endef check-packages: @$(foreach package,$(all_packages),$(call check_or_remove_cached,$(package));) check-sources: @$(foreach package,$(all_packages),$(call check_or_remove_sources,$(package));) $(host_prefix)/share/config.site: check-packages check-packages: check-sources clean-all: clean @rm -rf $(SOURCES_PATH) x86_64* i686* arm* aarch64* clean: @rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD) install: check-packages $(host_prefix)/share/config.site download-one: check-sources $(all_sources) download-osx: @$(MAKE) -s HOST=x86_64-apple-darwin16 download-one download-linux: @$(MAKE) -s HOST=x86_64-unknown-linux-gnu download-one download-win: @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one download: download-osx download-linux download-win build-linux64: download-linux @$(MAKE) -s HOST=x86_64-linux-gnu install build-linux32: download-linux @$(MAKE) -s HOST=i686-pc-linux-gnu install build-linux-arm: download-linux @$(MAKE) -s HOST=arm-linux-gnueabihf install build-linux-aarch64: download-linux @$(MAKE) -s HOST=aarch64-linux-gnu install build-osx: download-osx @$(MAKE) -s HOST=x86_64-apple-darwin16 install build-win32: download-win @$(MAKE) -s HOST=i686-w64-mingw32 install build-win64: download-win @$(MAKE) -s HOST=x86_64-w64-mingw32 install build-all: build-linux64 build-linux32 build-linux-arm build-linux-aarch64 build-osx build-win32 build-win64 $(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package)))) .PHONY: install cached clean clean-all download-one download-osx download-linux download-win download check-packages check-sources diff --git a/depends/README.md b/depends/README.md index a8d8f9f18..28fb6b7a8 100644 --- a/depends/README.md +++ b/depends/README.md @@ -1,117 +1,118 @@ ### Usage To build dependencies for the current arch+OS: make To build for another arch/OS: make build- Where `` is one of the following: - linux64 - linux32 - linux-arm - linux-aarch64 - osx - win64 For example, building the dependencies for macOS: make build-osx Note that it will use all the CPU cores available on the machine by default. This behavior can be changed by setting the `JOBS` environment variable (see below). To use the dependencies for building Bitcoin ABC, you need to set the platform file to be used by `cmake`. The platform files are located under `cmake/platforms/`. For example, cross-building for macOS (run from the project root): mkdir build_osx cd build_osx cmake -GNinja .. -DCMAKE_TOOLCHAIN_FILE=../cmake/platforms/OSX.cmake ninja No other options are needed, the paths are automatically configured. ### Install the required dependencies: Ubuntu & Debian #### Common to all arch/OS sudo apt-get install build-essential autoconf automake cmake curl git libtool ninja-build patch pkg-config python3 #### For macOS cross compilation sudo apt-get install imagemagick libbz2-dev libcap-dev librsvg2-bin libtiff-tools python3-setuptools #### For Win64 cross compilation - see [build-windows.md](../doc/build-windows.md#cross-compilation-for-ubuntu-and-windows-subsystem-for-linux) #### For linux cross compilation Common linux dependencies: sudo apt-get install gperf For linux 32 bits cross compilation: First add the i386 architecture to `dpkg`: sudo dpkg --add-architecture i386 sudo apt-get update Then install the dependencies: sudo apt-get install lib32stdc++-8-dev libc6-dev:i386 For linux ARM cross compilation: sudo apt-get install g++-arm-linux-gnueabihf For linux AARCH64 cross compilation: sudo apt-get install g++-aarch64-linux-gnu ### Dependency Options The following can be set when running make: make FOO=bar SOURCES_PATH: downloaded sources will be placed here BASE_CACHE: built packages will be placed here SDK_PATH: Path where sdk's can be found (used by macOS) FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up NO_QT: Don't download/build/cache qt and its dependencies NO_QR: Don't download/build/cache packages needed for enabling qrencode NO_ZMQ: Don't download/build/cache packages needed for enabling zeromq NO_WALLET: Don't download/build/cache libs needed to enable the wallet NO_UPNP: Don't download/build/cache packages needed for enabling upnp + NO_JEMALLOC: Don't download/build/cache jemalloc DEBUG: disable some optimizations and enable more runtime checking RAPIDCHECK: build rapidcheck (experimental, requires cmake) NO_PROTOBUF: Don't download/build/cache protobuf (used for BIP70 support) HOST_ID_SALT: Optional salt to use when generating host package ids BUILD_ID_SALT: Optional salt to use when generating build package ids JOBS: Number of jobs to use for each package build If some packages are not built, for example by building the depends with `make NO_WALLET=1`, the appropriate options should be set when building Bitcoin ABC using these dependencies. In this example, `-DBUILD_BITCOIN_WALLET=OFF` should be passed to the `cmake` command line to ensure that the build will not fail due to missing dependencies. Additional targets: download: run 'make download' to fetch all sources without building them download-osx: run 'make download-osx' to fetch all sources needed for macOS builds download-win: run 'make download-win' to fetch all sources needed for win builds download-linux: run 'make download-linux' to fetch all sources needed for linux builds build-all: build the dependencies for all the arch/OS ### Other documentation - [description.md](description.md): General description of the depends system - [packages.md](packages.md): Steps for adding packages diff --git a/depends/packages/jemalloc.mk b/depends/packages/jemalloc.mk new file mode 100644 index 000000000..aa0f03766 --- /dev/null +++ b/depends/packages/jemalloc.mk @@ -0,0 +1,21 @@ +package=jemalloc +$(package)_version=5.2.1 +$(package)_download_path=https://github.com/jemalloc/jemalloc/releases/download/$($(package)_version) +$(package)_file_name=jemalloc-$($(package)_version).tar.bz2 +$(package)_sha256_hash=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 + +define $(package)_set_vars +$(package)_config_opts=--disable-libdl --disable-shared --enable-cxx --enable-option-checking +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) -j$(JOBS) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index b53f783af..f3795b724 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,27 +1,29 @@ packages:=boost openssl libevent protobuf_native_packages = native_protobuf protobuf_packages = protobuf qt_packages = zlib qrencode_packages = qrencode qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig rapidcheck_packages = rapidcheck qt_darwin_packages=qt qt_mingw32_packages=qt wallet_packages=bdb zmq_packages=zeromq upnp_packages=miniupnpc darwin_native_packages = native_biplist native_ds_store native_mac_alias ifneq ($(build_os),darwin) darwin_native_packages += native_cctools native_cdrkit native_libdmg-hfsplus endif + +jemalloc_packages = jemalloc diff --git a/doc/dependencies.md b/doc/dependencies.md index 8c9385625..a8c8b1269 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -1,48 +1,49 @@ Dependencies ============ These are the dependencies currently used by Bitcoin ABC. You can find instructions for installing them in the `build-*.md` file for your platform. | Dependency | Version used | Minimum required | CVEs | Shared | [Bundled Qt library](https://doc.qt.io/qt-5/configure-options.html) | | --- | --- | --- | --- | --- | --- | | Berkeley DB | [5.3.28](http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 5.3 | No | | | | Boost | [1.70.0](https://www.boost.org/users/download/) | 1.58.0 | No | | | | Clang | | [3.4](https://releases.llvm.org/download.html) (C++14 support) | | | | | CMake | | [3.13](https://cmake.org/download/) | | | | | Expat | [2.2.7](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.6](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | | FreeType | [2.7.1](http://download.savannah.gnu.org/releases/freetype) | | No | | | | GCC | | [5.0](https://gcc.gnu.org/) (C++14 support) | | | | | HarfBuzz-NG | | | | | | +| jemalloc | [5.2.1](https://github.com/jemalloc/jemalloc/releases) | | | | | | libevent | [2.1.11-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | | libpng | | | | | Yes | | libsrvg | | | | | | | MiniUPnPc | [2.0.20180203](http://miniupnp.free.fr/files) | 1.5 | No | | | | Ninja | | [1.5.1](https://github.com/ninja-build/ninja/releases) | | | | | OpenSSL | [1.0.1k](https://www.openssl.org/source) | | Yes | | | | PCRE | | | | | Yes | | protobuf | [2.6.1](https://github.com/google/protobuf/releases) | | No | | | | Python (tests) | | [3.5](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | | Qt | [5.9.7](https://download.qt.io/official_releases/qt/) | 5.5.1 | No | | | | XCB | | | | | Yes (Linux only) | | xkbcommon | | | | | Yes (Linux only) | | ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.1.5 | No | | | | zlib | [1.2.11](http://zlib.net/) | | | | No | Controlling dependencies ------------------------ Some dependencies are not needed in all configurations. The following are some factors that affect the dependency list. #### Options passed to `cmake` * MiniUPnPc is not needed with `-DENABLE_UPNP=OFF`. * Berkeley DB is not needed with `-DBUILD_BITCOIN_WALLET=OFF`. * protobuf is not needed with `-DENABLE_BIP70=OFF`. * Qt is not needed with `-DBUILD_BITCOIN_QT=OFF`. * qrencode is not needed with `-DENABLE_QRCODE=OFF`. * ZeroMQ is not needed with the `-DBUILD_BITCOIN_ZMQ=OFF`. #### Other * librsvg is only needed if you need to run `ninja osx-dmg` on (cross-compilation to) macOS.