diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index aaa16e426..f7ce417d4 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -1,458 +1,458 @@ #!/usr/bin/env bash export LC_ALL=C set -e -o pipefail # Source the common prelude, which: # 1. Checks if we're at the top directory of the Bitcoin ABC repository # 2. Defines a few common functions and variables # # shellcheck source=libexec/prelude.bash source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" ################### ## SANITY CHECKS ## ################### ################ # Required non-builtin commands should be invocable ################ check_tools cat mkdir make getent curl git guix ################ # GUIX_BUILD_OPTIONS should be empty ################ # # GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that # can perform builds. This seems like what we want instead of # ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually # _appended_ to normal command-line options. Meaning that they will take # precedence over the command-specific ADDITIONAL_GUIX__FLAGS. # # This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's # existence here and direct users of this script to use our (more flexible) # custom environment variables. if [ -n "$GUIX_BUILD_OPTIONS" ]; then cat << EOF Error: Environment variable GUIX_BUILD_OPTIONS is not empty: '$GUIX_BUILD_OPTIONS' Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a specific guix command. See contrib/guix/README.md for more details. EOF exit 1 fi ################ # The git worktree should not be dirty ################ if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then cat << EOF ERR: The current git worktree is dirty, which may lead to broken builds. Aborting... Hint: To make your git worktree clean, You may want to: 1. Commit your changes, 2. Stash your changes, or 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on using a dirty worktree EOF exit 1 fi ################ # Build directories should not exist ################ # Default to building for all supported HOSTs (overridable by environment) # TODO: support x86_64-w64-mingw32, x86_64-apple-darwin, arm64-apple-darwin, # aarch64-linux-gnu, arm-linux-gnueabihf export HOSTS="${HOSTS:-x86_64-linux-gnu}" # Usage: distsrc_for_host HOST # # HOST: The current platform triple we're building for # distsrc_for_host() { echo "distsrc-${VERSION}-${1}" } # Accumulate a list of build directories that already exist... hosts_distsrc_exists="" for host in $HOSTS; do if [ -e "$(distsrc_for_host "$host")" ]; then hosts_distsrc_exists+=" ${host}" fi done if [ -n "$hosts_distsrc_exists" ]; then # ...so that we can print them out nicely in an error message cat << EOF ERR: Build directories already exist for the following platform triples you're attempting to build, probably because of previous builds. Please remove, or otherwise deal with them prior to starting another build. Aborting... Hint: To blow everything away, you may want to use: $ ./contrib/guix/guix-clean Specifically, this will remove all files without an entry in the index, excluding the SDK directory, the depends download cache, the depends built packages cache, the garbage collector roots for Guix environments, and the output directory. EOF for host in $hosts_distsrc_exists; do echo " ${host} '$(distsrc_for_host "$host")'" done exit 1 fi ################ # When building for darwin, the macOS SDK should exist ################ for host in $HOSTS; do case "$host" in *darwin*) - OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+= @@g')" + OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')" if [ -e "$OSX_SDK" ]; then echo "Found macOS SDK at '${OSX_SDK}', using..." break else echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..." exit 1 fi ;; esac done ################ # current disk should have enough space ################ avail_KiB="$(df -Pk . | sed 1d | tr -s ' ' | cut -d' ' -f4)" total_required_KiB=0 for host in $HOSTS; do case "$host" in *darwin*) required_KiB=440000 ;; *mingw*) required_KiB=7600000 ;; *) required_KiB=6400000 ;; esac total_required_KiB=$((total_required_KiB+required_KiB)) done if (( total_required_KiB > avail_KiB )); then total_required_GiB=$((total_required_KiB / 1048576)) avail_GiB=$((avail_KiB / 1048576)) echo "Performing a Bitcoin ABC Guix build for the selected HOSTS requires ${total_required_GiB} GiB, however, only ${avail_GiB} GiB is available. Please free up some disk space before performing the build." exit 1 fi ################ # Check that we can connect to the guix-daemon ################ cat << EOF Checking that we can connect to the guix-daemon... Hint: If this hangs, you may want to try turning your guix-daemon off and on again. EOF if ! guix gc --list-failures > /dev/null; then cat << EOF ERR: Failed to connect to the guix-daemon, please ensure that one is running and reachable. EOF exit 1 fi # Developer note: we could use `guix repl` for this check and run: # # (import (guix store)) (close-connection (open-connection)) # # However, the internal API is likely to change more than the CLI invocation ################ # Services database must have basic entries ################ if ! getent services http https ftp > /dev/null 2>&1; then cat << EOF ERR: Your system's C library cannot find service database entries for at least one of the following services: http, https, ftp. Hint: Most likely, /etc/services does not exist yet (common for docker images and minimal distros), or you don't have permissions to access it. If /etc/services does not exist yet, you may want to install the appropriate package for your distro which provides it. On Debian/Ubuntu: netbase On Arch Linux: iana-etc For more information, see: getent(1), services(5) EOF fi ######### # SETUP # ######### # Determine the maximum number of jobs to run simultaneously (overridable by # environment) JOBS="${JOBS:-$(nproc)}" # Usage: host_to_commonname HOST # # HOST: The current platform triple we're building for # host_to_commonname() { case "$1" in *darwin*) echo osx ;; *mingw*) echo win ;; *linux*) echo linux ;; *) exit 1 ;; esac } # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" # Precious directories are those which should not be cleaned between successive # guix builds depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH' precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE" # Usage: contains IFS-SEPARATED-LIST ITEM contains() { for i in ${1}; do if [ "$i" = "${2}" ]; then return 0 # Found! fi done return 1 } # If the user explicitly specified a precious directory, create it so we # can map it into the container for precious_dir_name in $precious_dir_names; do precious_dir_path="${!precious_dir_name}" if [ -n "$precious_dir_path" ]; then if [ ! -e "$precious_dir_path" ]; then mkdir -p "$precious_dir_path" elif [ -L "$precious_dir_path" ]; then echo "ERR: ${precious_dir_name} cannot be a symbolic link" exit 1 elif [ ! -d "$precious_dir_path" ]; then echo "ERR: ${precious_dir_name} must be a directory" exit 1 fi fi done mkdir -p "$VAR_BASE" # Record the _effective_ values of precious directories such that guix-clean can # avoid clobbering them if appropriate. # # shellcheck disable=SC2046,SC2086 { # Get depends precious dir definitions from depends make -C "${PWD}/depends" \ --no-print-directory \ -- $(printf "print-%s\n" $depends_precious_dir_names) # Get remaining precious dir definitions from the environment for precious_dir_name in $precious_dir_names; do precious_dir_path="${!precious_dir_name}" if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then echo "${precious_dir_name}=${precious_dir_path}" fi done } > "${VAR_BASE}/precious_dirs" # Make sure an output directory exists for our builds OUTDIR_BASE="${OUTDIR_BASE:output}" mkdir -p "$OUTDIR_BASE" # Download the depends sources now as we won't have internet access in the build # container for host in $HOSTS; do make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} done # Usage: outdir_for_host HOST SUFFIX # # HOST: The current platform triple we're building for # outdir_for_host() { echo "${OUTDIR_BASE}/${1}${2:+-${2}}" } # Usage: profiledir_for_host HOST SUFFIX # # HOST: The current platform triple we're building for # profiledir_for_host() { echo "${PROFILES_BASE}/${1}${2:+-${2}}" } ######### # BUILD # ######### # Function to be called when building for host ${1} and the user interrupts the # build int_trap() { cat << EOF ** INT received while building ${1}, you may want to clean up the relevant work directories (e.g. distsrc-*) before rebuilding Hint: To blow everything away, you may want to use: $ ./contrib/guix/guix-clean Specifically, this will remove all files without an entry in the index, excluding the SDK directory, the depends download cache, the depends built packages cache, the garbage collector roots for Guix environments, and the output directory. EOF } # Deterministically build Bitcoin ABC # shellcheck disable=SC2153 for host in $HOSTS; do # Display proper warning when the user interrupts the build trap 'int_trap ${host}' INT ( # Required for 'contrib/guix/manifest.scm' to output the right manifest # for the particular $HOST we're building for export HOST="$host" # shellcheck disable=SC2030 cat << EOF INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} ...running at most ${JOBS:?not set} jobs ...from worktree directory: '${PWD}' ...bind-mounted in container to: '/bitcoin' ...in build directory: '$(distsrc_for_host "$HOST")' ...bind-mounted in container to: /bitcoin/$(distsrc_for_host "$HOST") ...outputting in: '$(outdir_for_host "$HOST")' ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' EOF # Run the build script 'contrib/guix/libexec/build.sh' in the build # container specified by 'contrib/guix/manifest.scm'. # # Explanation of `guix environment` flags: # # --container run command within an isolated container # # Running in an isolated container minimizes build-time differences # between machines and improves reproducibility # # --pure unset existing environment variables # # Same rationale as --container # # --no-cwd do not share current working directory with an # isolated container # # When --container is specified, the default behavior is to share # the current working directory with the isolated container at the # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to # '/home/satoshi/bitcoin/'). This means that the $PWD inside the # container becomes a source of irreproducibility. --no-cwd disables # this behaviour. # # --share=SPEC for containers, share writable host file system # according to SPEC # # --share="$PWD"=/bitcoin # # maps our current working directory to /bitcoin # inside the isolated container, which we later cd # into. # # While we don't want to map our current working directory to the # same exact path (as this introduces irreproducibility), we do want # it to be at a _fixed_ path _somewhere_ inside the isolated # container so that we have something to build. '/bitcoin' was # chosen arbitrarily. # # ${SOURCES_PATH:+--share="$SOURCES_PATH"} # # make the downloaded depends sources path available # inside the isolated container # # The isolated container has no network access as it's in a # different network namespace from the main machine, so we have to # make the downloaded depends sources available to it. The sources # should have been downloaded prior to this invocation. # # --keep-failed keep build tree of failed builds # # When builds of the Guix environment itself (not Bitcoin ABC) # fail, it is useful for the build tree to be kept for debugging # purposes. # # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} # # fetch substitute from SUBSTITUTE_URLS if they are # authorized # # Depending on the user's security model, it may be desirable to use # substitutes (pre-built packages) from servers that the user trusts. # Please read the README.md in the same directory as this file for # more information. # # shellcheck disable=SC2086,SC2031 time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ --container \ --pure \ --no-cwd \ --share="$PWD"=/bitcoin \ --share="$OUTDIR_BASE"=/outdir-base \ --expose="$(git rev-parse --git-common-dir)" \ ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ ${BASE_CACHE:+--share="$BASE_CACHE"} \ ${SDK_PATH:+--share="$SDK_PATH"} \ --cores="$JOBS" \ --keep-failed \ --fallback \ --link-profile \ --root="$(profiledir_for_host "${HOST}")" \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ DISTNAME="$DISTNAME" \ JOBS="$JOBS" \ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ ${V:+V=1} \ ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ DISTSRC="$(distsrc_for_host "$HOST")" \ OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" ) done diff --git a/depends/Makefile b/depends/Makefile index 41329c63f..b7614d914 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -1,247 +1,247 @@ .NOTPARALLEL : # Pattern rule to print variables, e.g. make print-build_arch print-%: - @echo '$*' = '$($*)' + @echo '$*'='$($*)' # When invoking a sub-make, keep only the command line variable definitions # matching the pattern in the filter function. # # e.g. invoking: # $ make A=1 C=1 print-MAKEOVERRIDES print-MAKEFLAGS # # with the following in the Makefile: # MAKEOVERRIDES := $(filter A=% B=%,$(MAKEOVERRIDES)) # # will print: # MAKEOVERRIDES = A=1 # MAKEFLAGS = -- A=1 # # this is because as the GNU make manual says: # The command line variable definitions really appear in the variable # MAKEOVERRIDES, and MAKEFLAGS contains a reference to this variable. # # and since the GNU make manual also says: # variables defined on the command line are passed to the sub-make through # MAKEFLAGS # # this means that sub-makes will be invoked as if: # $(MAKE) A=1 blah blah MAKEOVERRIDES := $(filter V=%,$(MAKEOVERRIDES)) SOURCES_PATH ?= $(BASEDIR)/sources WORK_PATH = $(BASEDIR)/work BASE_CACHE ?= $(BASEDIR)/built SDK_PATH ?= $(BASEDIR)/SDKs NO_QT ?= NO_PROTOBUF ?= NO_QR ?= NO_WALLET ?= NO_ZMQ ?= NO_UPNP ?= NO_NATPMP ?= NO_JEMALLOC ?= FALLBACK_DOWNLOAD_PATH ?= https://download.bitcoinabc.org/depends-sources BUILD = $(shell ./config.guess) HOST ?= $(BUILD) PATCHES_PATH = $(BASEDIR)/patches BASEDIR = $(CURDIR) HASH_LENGTH:=11 DOWNLOAD_CONNECT_TIMEOUT:=30 DOWNLOAD_RETRIES:=3 HOST_ID_SALT ?= salt BUILD_ID_SALT ?= salt JOBS ?= $(shell echo $$(($(shell nproc 2> /dev/null || sysctl -n hw.ncpu 2> /dev/null || echo 0) + 1))) host:=$(BUILD) ifneq ($(HOST),) host:=$(HOST) host_toolchain:=$(HOST)- endif ifneq ($(DEBUG),) release_type=debug else release_type=release endif base_build_dir=$(WORK_PATH)/build base_staging_dir=$(WORK_PATH)/staging base_download_dir=$(WORK_PATH)/download canonical_host:=$(shell ./config.sub $(HOST)) build:=$(shell ./config.sub $(BUILD)) build_arch =$(firstword $(subst -, ,$(build))) build_vendor=$(word 2,$(subst -, ,$(build))) full_build_os:=$(subst $(build_arch)-$(build_vendor)-,,$(build)) build_os:=$(findstring linux,$(full_build_os)) build_os+=$(findstring darwin,$(full_build_os)) build_os:=$(strip $(build_os)) ifeq ($(build_os),) build_os=$(full_build_os) endif host_arch=$(firstword $(subst -, ,$(canonical_host))) host_vendor=$(word 2,$(subst -, ,$(canonical_host))) full_host_os:=$(subst $(host_arch)-$(host_vendor)-,,$(canonical_host)) host_os:=$(findstring linux,$(full_host_os)) host_os+=$(findstring darwin,$(full_host_os)) host_os+=$(findstring mingw32,$(full_host_os)) host_os:=$(strip $(host_os)) ifeq ($(host_os),) host_os=$(full_host_os) endif $(host_arch)_$(host_os)_prefix=$(BASEDIR)/$(host) $(host_arch)_$(host_os)_host=$(host) host_prefix=$($(host_arch)_$(host_os)_prefix) build_prefix=$(host_prefix)/native build_host=$(build) AT_$(V):= AT_:=@ AT:=$(AT_$(V)) all: install include hosts/$(host_os).mk include hosts/default.mk include builders/$(build_os).mk include builders/default.mk include packages/packages.mk # Previously, we directly invoked the well-known programs using $(shell ...) # to contruct build_id_string. However, that was problematic because: # # 1. When invoking a shell, GNU Make special-cases exit code 127 (command not # found) by not capturing the output but instead passing it through. This is # not done for any other exit code. # # 2. Characters like '#' (from these programs' output) would end up in make # variables like build_id_string, which would be wrongly interpreted by make # when these variables were used. # # Therefore, we should avoid having arbitrary strings in make variables where # possible. The gen_id script used here hashes the output to construct a # "make-safe" id. # # Also note that these lines need to be: # # 1. After including {hosts,builders}/*.mk, since they rely on the tool # variables (e.g. build_CC, host_STRIP, etc.) to be set. # # 2. Before including packages/*.mk (excluding packages/packages.mk), since # they rely on the build_id variables # build_id:=$(shell env CC='$(build_CC)' CXX='$(build_CXX)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') $(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' CXX='$(host_CXX)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') qrencode_packages_$(NO_QR) = $(qrencode_packages) qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) $(qrencode_packages_) bdb_packages_$(NO_BDB) = $(bdb_packages) sqlite_packages_$(NO_SQLITE) = $(sqlite_packages) wallet_packages_$(NO_WALLET) = $(bdb_packages_) $(sqlite_packages_) upnp_packages_$(NO_UPNP) = $(upnp_packages) natpmp_packages_$(NO_NATPMP) = $(natpmp_packages) zmq_packages_$(NO_ZMQ) = $(zmq_packages) protobuf_packages_$(NO_PROTOBUF) = $(protobuf_packages) jemalloc_packages_$(NO_JEMALLOC) = $(jemalloc_packages) packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) ifneq ($(protobuf_packages_),) native_packages += $(protobuf_native_packages) packages += $(protobuf_packages) endif ifneq ($(zmq_packages_),) packages += $(zmq_packages) endif ifneq ($(jemalloc_packages_),) packages += $(jemalloc_packages) endif all_packages = $(packages) $(native_packages) meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk $(host_arch)_$(host_os)_native_binutils?=$($(host_os)_native_binutils) $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) include funcs.mk final_build_id_long+=$(shell $(build_SHA256SUM) config.sub) final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)rm -rf $(@D) $(AT)mkdir -p $(@D) $(AT)echo copying packages: $^ $(AT)echo to: $(@D) $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) $(AT)touch $@ define check_or_remove_cached mkdir -p $(BASE_CACHE)/$(host)/$(package) && cd $(BASE_CACHE)/$(host)/$(package); \ $(build_SHA256SUM) -c $($(package)_cached_checksum) >/dev/null 2>/dev/null || \ ( rm -f $($(package)_cached_checksum); \ if test -f "$($(package)_cached)"; then echo "Checksum mismatch for $(package). Forcing rebuild.."; rm -f $($(package)_cached_checksum) $($(package)_cached); fi ) endef define check_or_remove_sources mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \ test -f $($(package)_fetched) && ( $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \ ( echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; \ rm -f $($(package)_all_sources) $($(1)_fetched))) || true endef check-packages: @$(foreach package,$(all_packages),$(call check_or_remove_cached,$(package));) check-sources: @$(foreach package,$(all_packages),$(call check_or_remove_sources,$(package));) check-packages: check-sources clean-all: clean @rm -rf $(SOURCES_PATH) x86_64* i686* arm* aarch64* clean: @rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD) install: check-packages $(host_prefix)/.stamp_$(final_build_id) download-one: check-sources $(all_sources) download-osx: @$(MAKE) -s HOST=x86_64-apple-darwin19 download-one download-linux: @$(MAKE) -s HOST=x86_64-unknown-linux-gnu download-one download-win: @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one download: download-osx download-linux download-win build-linux64: download-linux @$(MAKE) -s HOST=x86_64-linux-gnu install build-linux32: download-linux @$(MAKE) -s HOST=i686-pc-linux-gnu install build-linux-arm: download-linux @$(MAKE) -s HOST=arm-linux-gnueabihf install build-linux-aarch64: download-linux @$(MAKE) -s HOST=aarch64-linux-gnu install build-osx: download-osx @$(MAKE) -s HOST=x86_64-apple-darwin19 install build-win64: download-win @$(MAKE) -s HOST=x86_64-w64-mingw32 install build-all: build-linux64 build-linux32 build-linux-arm build-linux-aarch64 build-osx build-win64 $(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package)))) .PHONY: install cached clean clean-all download-one download-osx download-linux download-win download check-packages check-sources