diff --git a/contrib/gitian-builder/bin/gbuild b/contrib/gitian-builder/bin/gbuild index 2a9def4f3..a8b2c0cdc 100755 --- a/contrib/gitian-builder/bin/gbuild +++ b/contrib/gitian-builder/bin/gbuild @@ -1,393 +1,409 @@ #!/usr/bin/env ruby require 'optparse' require 'yaml' require 'fileutils' require 'pathname' @options = {:num_procs => 2, :memory => 2000} @bitness = { 'i386' => 32, 'amd64' => 64, 'linux64' => 64, } @arches = { 'i386' => 'i386', 'amd64' => 'x86_64', 'linux64' => 'linux64', } def system!(cmd) system(cmd) or raise "failed to run #{cmd}" end def sanitize(str, where) raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/) str end def sanitize_path(str, where) raise "unsanitary string in #{where}" if (str =~ /[^@\w\/.:+-]/) str end def info(str) puts str unless @options[:quiet] end def build_one_configuration(suite, arch, build_desc) FileUtils.rm_f("var/install.log") FileUtils.rm_f("var/build.log") bits = @bitness[arch] or raise "unknown architecture ${arch}" if ENV["USE_LXC"] ENV["LXC_ARCH"] = arch ENV["LXC_SUITE"] = suite end suitearch = "#{suite}-#{arch}" info "Stopping target if it is up" system "stop-target" sleep 1 unless @options[:skip_image] info "Making a new image copy" system! "make-clean-vm --suite #{suite} --arch #{arch}" end info "Starting target" system! "start-target #{bits} #{suitearch}&" $stdout.write "Checking if target is up" (1..30).each do system "on-target true 2> /dev/null" and break sleep 2 $stdout.write '.' end info '' system! "on-target true" system! "on-target -u root tee -a /etc/sudoers.d/#{ENV['DISTRO'] || 'ubuntu'} > /dev/null << EOF %#{ENV['DISTRO'] || 'ubuntu'} ALL=(ALL) NOPASSWD: ALL EOF" if build_desc["sudo"] and @options[:allow_sudo] info "Preparing build environment" system! "on-target setarch #{@arches[arch]} bash < target-bin/init-build.sh" build_desc["files"].each do |filename| filename = sanitize(filename, "files section") system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/" end if build_desc["enable_cache"] if File.directory?("cache/#{build_desc["name"]}") system! "copy-to-target #{@quiet_flag} cache/#{build_desc["name"]}/ cache/" end if File.directory?("cache/common") system! "copy-to-target #{@quiet_flag} cache/common/ cache/" end end if build_desc["multiarch"] info "Adding multiarch support (log in var/install.log)" for a in build_desc["multiarch"] system! "on-target -u root dpkg --add-architecture #{a} >> var/install.log 2>&1" end end + if build_desc["repositories"] + info "Adding repositories to the sources list (log in var/install.log)" + for r in build_desc["repositories"] + system! "on-target -u root tee -a /etc/apt/sources.list >> var/install.log 2>&1 << EOF +#{r["source"]} +EOF" + end + end + info "Updating apt-get repository (log in var/install.log)" system! "on-target -u root apt-get update >> var/install.log 2>&1" info "Installing additional packages (log in var/install.log)" system! "on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install #{build_desc["packages"].join(" ")} >> var/install.log 2>&1" + if build_desc["repositories"] + for r in build_desc["repositories"] + info "Installing additional packages from repository #{r["distribution"]} (log in var/install.log)" + system! "on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get -t #{r["distribution"]} --no-install-recommends -y install #{r["packages"].join(" ")} >> var/install.log 2>&1" + end + end + if build_desc["alternatives"] info "Set alternatives (log in var/install.log)" for a in build_desc["alternatives"] system! "on-target -u root update-alternatives --set #{a["package"]} #{a["path"]} >> var/install.log 2>&1" end end if @options[:upgrade] || system("on-target -u root '[ ! -e /var/cache/gitian/initial-upgrade ]'") info "Upgrading system, may take a while (log in var/install.log)" system! "on-target -u root bash < target-bin/upgrade-system.sh >> var/install.log 2>&1" end info "Creating package manifest" system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest" info "Creating build script (var/build-script)" File.open("var/build-script", "w") do |script| script.puts "#!/bin/bash" script.puts "set -e" script.puts "export LANG='en_US.UTF-8'" script.puts "export LC_ALL='en_US.UTF-8'" script.puts "umask 002" script.puts "export OUTDIR=$HOME/out" script.puts "GBUILD_BITS=#{bits}" if build_desc["enable_cache"] script.puts "GBUILD_CACHE_ENABLED=1" script.puts "GBUILD_PACKAGE_CACHE=$HOME/cache/#{build_desc["name"]}" script.puts "GBUILD_COMMON_CACHE=$HOME/cache/common" end script.puts "MAKEOPTS=(-j#{@options[:num_procs]})" script.puts author_date = nil build_desc["remotes"].each do |remote| dir = sanitize(remote["dir"], remote["dir"]) author_date = `cd inputs/#{dir} && TZ=UTC git log --date='format-local:%F %T' --format="%ad" -1`.strip raise "error looking up author date in #{dir}" unless $?.exitstatus == 0 system! "copy-to-target #{@quiet_flag} inputs/#{dir} build/" script.puts "(cd build/#{dir} && git reset -q --hard && git clean -q -f -d)" end script.puts ref_datetime = build_desc["reference_datetime"] || author_date (ref_date, ref_time) = ref_datetime.split script.puts "REFERENCE_DATETIME='#{ref_datetime}'" script.puts "REFERENCE_DATE='#{ref_date}'" script.puts "REFERENCE_TIME='#{ref_time}'" script.puts script.puts "cd build" script.puts build_desc["script"] end info "Running build script (log in var/build.log)" if ENV["CI"] system! "on-target setarch #{@arches[arch]} bash -x < var/build-script 2>&1 | tee var/build.log" else system! "on-target setarch #{@arches[arch]} bash -x < var/build-script > var/build.log 2>&1" end end ################################ OptionParser.new do |opts| opts.banner = "Usage: build [options] .yml" opts.on("--allow-sudo", "override SECURITY on the target VM and allow the use of sudo with no password for the default user") do |v| @options[:allow_sudo] = v end opts.on("-i", "--skip-image", "reuse current target image") do |v| @options[:skip_image] = v end opts.on("--upgrade", "upgrade guest with latest packages") do |v| @options[:upgrade] = v end opts.on("-q", "--quiet", "be quiet") do |v| @options[:quiet] = v end opts.on("-j PROCS", "--num-make PROCS", "number of processes to use") do |v| @options[:num_procs] = v end opts.on("-m MEM", "--memory MEM", "memory to allocate in MiB") do |v| @options[:memory] = v end opts.on("-c PAIRS", "--commit PAIRS", "comma separated list of DIRECTORY=COMMIT pairs") do |v| @options[:commit] = v end opts.on("-u PAIRS", "--url PAIRS", "comma separated list of DIRECTORY=URL pairs") do |v| @options[:url] = v end opts.on("-o", "--cache-read-only", "only use existing cache files, do not update them") do |v| @options[:cache_ro] = v end opts.on("--skip-fetch", "skip fetching the latest git objects and refs from the remote source") do |v| @options[:skip_fetch] = v end end.parse! if !ENV["USE_LXC"] and !ENV["USE_DOCKER"] and !ENV["USE_VBOX"] and !File.exist?("/dev/kvm") $stderr.puts "\n************* WARNING: kvm not loaded, this will probably not work out\n\n" end base_dir = Pathname.new(__FILE__).expand_path.dirname.parent libexec_dir = base_dir + 'libexec' ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH'] ENV['GITIAN_BASE'] = base_dir.to_s ENV['NPROCS'] = @options[:num_procs].to_s ENV['VMEM'] = @options[:memory].to_s @quiet_flag = @options[:quiet] ? "-q" : "" build_desc_file = ARGV.shift or raise "must supply YAML build description file" build_desc = YAML.load_file(build_desc_file) in_sums = [] build_dir = 'build' result_dir = 'result' cache_dir = 'cache' enable_cache = build_desc["enable_cache"] FileUtils.rm_rf(build_dir) FileUtils.mkdir(build_dir) FileUtils.mkdir_p(result_dir) package_name = build_desc["name"] or raise "must supply name" package_name = sanitize(package_name, "package name") if enable_cache FileUtils.mkdir_p(File.join(cache_dir, "common")) FileUtils.mkdir_p(File.join(cache_dir, package_name)) end distro = build_desc["distro"] || "ubuntu" suites = build_desc["suites"] or raise "must supply suites" archs = build_desc["architectures"] or raise "must supply architectures" build_desc["reference_datetime"] or build_desc["remotes"].size > 0 or raise "must supply `reference_datetime` or `remotes`" docker_image_digests = build_desc["docker_image_digests"] || [] # if docker_image_digests are supplied, it must be the same length as suites if docker_image_digests.size > 0 and suites.size != docker_image_digests.size raise "`suites` and `docker_image_digests` must both be the same size if both are supplied" elsif ENV["USE_DOCKER"] and docker_image_digests.size > 0 and suites.size == docker_image_digests.size suites = docker_image_digests end ENV['DISTRO'] = distro desc_sum = `sha256sum #{build_desc_file}` desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml") in_sums << desc_sum build_desc["files"].each do |filename| filename = sanitize(filename, "files section") in_sums << `cd inputs && sha256sum #{filename}` end commits = {} if @options[:commit] @options[:commit].split(',').each do |pair| (dir, commit) = pair.split('=') commits[dir] = commit end end urls = {} if @options[:url] @options[:url].split(',').each do |pair| (dir, url) = pair.split('=') urls[dir] = url end end build_desc["remotes"].each do |remote| if !remote["commit"] remote["commit"] = commits[remote["dir"]] raise "must specify a commit for directory #{remote["dir"]}" unless remote["commit"] end if urls[remote["dir"]] remote["url"] = urls[remote["dir"]] end dir = sanitize(remote["dir"], remote["dir"]) commit = sanitize(remote["commit"], remote["commit"]) unless File.exist?("inputs/#{dir}") system!("git init inputs/#{dir}") end if !@options[:skip_fetch] system!("cd inputs/#{dir} && git fetch --update-head-ok #{sanitize_path(remote["url"], remote["url"])} +refs/tags/*:refs/tags/* +refs/heads/*:refs/heads/*") system!("cd inputs/#{dir} && git checkout -q #{commit}") system!("cd inputs/#{dir} && git submodule update --init --recursive --force") end commit = `cd inputs/#{dir} && git log --format=%H -1 #{commit}`.strip raise "error looking up commit for tag #{remote["commit"]}" unless $?.exitstatus == 0 in_sums << "git:#{commit} #{dir}" end base_manifests = YAML::Omap.new suites.each do |suite| suite = sanitize(suite, "suite") archs.each do |arch| info "--- Building for #{suite} #{arch} ---" arch = sanitize(arch, "architecture") # Build! build_one_configuration(suite, arch, build_desc) info "Grabbing results from target" system! "copy-from-target #{@quiet_flag} out #{build_dir}" if enable_cache && !@options[:cache_ro] info "Grabbing cache from target" system! "copy-from-target #{@quiet_flag} cache/#{package_name}/ #{cache_dir}" system! "copy-from-target #{@quiet_flag} cache/common/ #{cache_dir}" end base_manifest = File.read("var/base-#{suite}-#{arch}.manifest") base_manifests["#{suite}-#{arch}"] = base_manifest end end out_dir = File.join(build_dir, "out") out_sums = {} cache_common_dir = File.join(cache_dir, "common") cache_package_dir = File.join(cache_dir, "#{package_name}") cache_common_sums = {} cache_package_sums = {} info "Generating report" Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out| next if File.directory?(file_in_out) file = file_in_out.sub(out_dir + File::SEPARATOR, '') file = sanitize_path(file, file_in_out) out_sums[file] = `cd #{out_dir} && sha256sum #{file}` raise "failed to sum #{file}" unless $? == 0 puts out_sums[file] unless @options[:quiet] end if enable_cache Dir.glob(File.join(cache_common_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out| next if File.directory?(file_in_out) file = file_in_out.sub(cache_common_dir + File::SEPARATOR, '') file = sanitize_path(file, file_in_out) cache_common_sums[file] = `cd #{cache_common_dir} && sha256sum #{file}` raise "failed to sum #{file}" unless $? == 0 end Dir.glob(File.join(cache_package_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out| next if File.directory?(file_in_out) file = file_in_out.sub(cache_package_dir + File::SEPARATOR, '') file = sanitize_path(file, file_in_out) cache_package_sums[file] = `cd #{cache_package_dir} && sha256sum #{file}` raise "failed to sum #{file}" unless $? == 0 end end out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('') in_manifest = in_sums.join('') cache_common_manifest = cache_common_sums.keys.sort.map { |key| cache_common_sums[key] }.join('') cache_package_manifest = cache_package_sums.keys.sort.map { |key| cache_package_sums[key] }.join('') # Use Omap to keep result deterministic report = YAML::Omap[ 'out_manifest', out_manifest, 'in_manifest', in_manifest, 'base_manifests', base_manifests, 'cache_common_manifest', cache_common_manifest, 'cache_package_manifest', cache_package_manifest, ] result_file = "#{package_name}-res.yml" File.open(File.join(result_dir, result_file), "w") do |io| io.write report.to_yaml end system!("cd #{result_dir} && sha256sum #{result_file}") unless @options[:quiet] info "Done." diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 7ed061cfd..bc0915851 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,193 +1,197 @@ --- name: "bitcoin-abc-linux" enable_cache: true distro: "debian" suites: - "buster" architectures: - "amd64" multiarch: - "i386" packages: - "autoconf" - "automake" - "binutils-aarch64-linux-gnu" - "binutils-arm-linux-gnueabihf" - "binutils-gold" - "bsdmainutils" - "build-essential" - "ca-certificates" -- "cmake" - "curl" - "faketime" - "g++-aarch64-linux-gnu" - "g++-arm-linux-gnueabihf" - "gcc-aarch64-linux-gnu" - "gcc-arm-linux-gnueabihf" - "git" - "gperf" - "lib32stdc++-8-dev" - "libc6-dev:i386" - "libtool" - "ninja-build" - "pkg-config" - "python3" # FIXME: these dependencies are only required to make CMake happy when building # native tools. They can be removed when the `NativeExecutable.cmake` gets # improved to avoid requiring all the bitcoin-abc dependencies. - "libboost-all-dev" - "libevent-dev" - "libssl-dev" +repositories: +- "distribution": "buster-backports" + "source": "deb http://deb.debian.org/debian/ buster-backports main" + packages: + - "cmake" remotes: - "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git" "dir": "bitcoin" files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu" # CMake toolchain file name differ from host name declare -A CMAKE_TOOLCHAIN_FILE CMAKE_TOOLCHAIN_FILE[i686-pc-linux-gnu]=Linux32.cmake CMAKE_TOOLCHAIN_FILE[x86_64-linux-gnu]=Linux64.cmake CMAKE_TOOLCHAIN_FILE[arm-linux-gnueabihf]=LinuxARM.cmake CMAKE_TOOLCHAIN_FILE[aarch64-linux-gnu]=LinuxAArch64.cmake # Allow extra cmake option to be specified for each host declare -A CMAKE_EXTRA_OPTIONS # ARM assembly is supported but experimental, disable it for the release CMAKE_EXTRA_OPTIONS[arm-linux-gnueabihf]="-DSECP256K1_USE_ASM=OFF" INSTALL_COMPONENTS="bitcoind bitcoin-qt bitcoin-seeder" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="date ar ranlib nm" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" HOST_LDFLAGS=-static-libstdc++ export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export GZIP="-9n" export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" create_per-host_faketime_wrappers "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} cd bitcoin SOURCEDIR=`pwd` BASEPREFIX=`pwd`/depends # Build dependencies for each host for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done # Faketime for binaries export PATH=${PATH_orig} create_global_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} mkdir -p source_package pushd source_package cmake -GNinja .. \ -DBUILD_BITCOIN_WALLET=OFF \ -DBUILD_BITCOIN_ZMQ=OFF \ -DBUILD_BITCOIN_SEEDER=OFF \ -DBUILD_BITCOIN_CLI=OFF \ -DBUILD_BITCOIN_TX=OFF \ -DBUILD_BITCOIN_QT=OFF \ -DBUILD_LIBBITCOINCONSENSUS=OFF \ -DENABLE_CLANG_TIDY=OFF \ -DENABLE_QRCODE=OFF \ -DENABLE_UPNP=OFF \ -DUSE_JEMALLOC=OFF ninja package_source SOURCEDIST=`echo bitcoin-abc-*.tar.gz` mv ${SOURCEDIST} .. popd DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` # Correct tar file order mkdir -p temp pushd temp tar xf ../$SOURCEDIST find bitcoin-abc-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST popd ORIGPATH="$PATH" # Extract the release tarball into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH=`pwd`/installed/${DISTNAME} mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST cmake -GNinja .. \ -DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${i}]} \ -DCLIENT_VERSION_IS_RELEASE=ON \ -DENABLE_CLANG_TIDY=OFF \ -DENABLE_REDUCE_EXPORTS=ON \ -DENABLE_STATIC_LIBSTDCXX=ON \ -DENABLE_GLIBC_BACK_COMPAT=ON \ -DCMAKE_INSTALL_PREFIX=${INSTALLPATH} \ -DCCACHE=OFF \ -DUSE_LD_GOLD=OFF \ ${CMAKE_EXTRA_OPTIONS[${i}]} ninja ninja security-check ninja symbol-check for _component in ${INSTALL_COMPONENTS}; do cmake -DCOMPONENT=${_component} -P cmake_install.cmake done cd installed find ${DISTNAME}/bin -type f -executable -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; find ${DISTNAME}/lib -type f -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz cd ../../ rm -rf distsrc-${i} done mkdir -p $OUTDIR/src mv $SOURCEDIST $OUTDIR/src diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index e86675e91..9c5ccfd8c 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,207 +1,211 @@ --- name: "bitcoin-abc-osx" enable_cache: true distro: "debian" suites: - "buster" architectures: - "amd64" packages: - "autoconf" - "automake" - "bsdmainutils" - "ca-certificates" -- "cmake" - "curl" - "faketime" - "fonts-tuffy" - "g++" - "git" - "imagemagick" - "libbz2-dev" - "libcap-dev" - "librsvg2-bin" - "libtiff-tools" - "libtinfo5" - "libtool" - "libz-dev" - "ninja-build" - "pkg-config" - "python3" - "python3-dev" - "python3-setuptools" # FIXME: these dependencies are only required to make CMake happy when building # native tools. They can be removed when the `NativeExecutable.cmake` gets # improved to avoid requiring all the bitcoin-abc dependencies. - "libboost-all-dev" - "libevent-dev" - "libssl-dev" +repositories: +- "distribution": "buster-backports" + "source": "deb http://deb.debian.org/debian/ buster-backports main" + packages: + - "cmake" remotes: - "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git" "dir": "bitcoin" files: - "MacOSX10.14.sdk.tar.gz" script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin16" # CMake toolchain file name differ from host name declare -A CMAKE_TOOLCHAIN_FILE CMAKE_TOOLCHAIN_FILE[x86_64-apple-darwin16]=OSX.cmake INSTALL_COMPONENTS="bitcoind bitcoin-qt bitcoin-seeder" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="ar ranlib date dmg genisoimage" export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export GZIP="-9n" export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi export ZERO_AR_DATE=1 function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" create_per-host_faketime_wrappers "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} cd bitcoin SOURCEDIR=`pwd` BASEPREFIX=`pwd`/depends mkdir -p ${BASEPREFIX}/SDKs tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz # Build dependencies for each host for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done # Faketime for binaries export PATH=${PATH_orig} create_global_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} mkdir -p source_package pushd source_package cmake -GNinja .. \ -DBUILD_BITCOIN_WALLET=OFF \ -DBUILD_BITCOIN_ZMQ=OFF \ -DBUILD_BITCOIN_ZMQ=OFF \ -DBUILD_BITCOIN_SEEDER=OFF \ -DBUILD_BITCOIN_CLI=OFF \ -DBUILD_BITCOIN_TX=OFF \ -DBUILD_BITCOIN_QT=OFF \ -DBUILD_LIBBITCOINCONSENSUS=OFF \ -DENABLE_CLANG_TIDY=OFF \ -DENABLE_QRCODE=OFF \ -DENABLE_UPNP=OFF \ -DUSE_JEMALLOC=OFF ninja package_source SOURCEDIST=`echo bitcoin-abc-*.tar.gz` mv ${SOURCEDIST} .. popd DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` # Correct tar file order mkdir -p temp pushd temp tar xf ../$SOURCEDIST find bitcoin-abc-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST popd ORIGPATH="$PATH" # Extract the release tarball into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH=`pwd`/installed/${DISTNAME} mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST cmake -GNinja .. \ -DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${i}]} \ -DCLIENT_VERSION_IS_RELEASE=ON \ -DENABLE_CLANG_TIDY=OFF \ -DENABLE_REDUCE_EXPORTS=ON \ -DCMAKE_INSTALL_PREFIX=${INSTALLPATH} \ -DCCACHE=OFF \ -DGENISOIMAGE_EXECUTABLE="${WRAP_DIR}/genisoimage" ninja for _component in ${INSTALL_COMPONENTS}; do cmake -DCOMPONENT=${_component} -P cmake_install.cmake done export PYTHONPATH="${BASEPREFIX}/${i}/native/lib/python3/dist-packages:${PYTHONPATH}" ninja osx-deploydir OSX_VOLNAME="$(cat osx_volname)" mkdir -p unsigned-app-${i} cp osx_volname unsigned-app-${i}/ cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i} cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff mv dist unsigned-app-${i} pushd unsigned-app-${i} find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz popd ninja osx-dmg ${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg cd installed # The install/strip target should do exactly that. However there is a bug in # in the Buster CMake version that causes strip to be called with the wrong # parameters when an Apple strip tool is used. This is fixed starting with # CMake versions >= 3.14, see: # https://gitlab.kitware.com/cmake/cmake/merge_requests/2892 find -path "*.app*" -type f -executable -exec mv {} ${DISTNAME}/bin/bitcoin-qt \; find ${DISTNAME}/bin -type f -executable -exec ${i}-strip -u -r {} \; find ${DISTNAME}/lib -type f -executable -exec ${i}-strip -x {} \; find ${DISTNAME} -not -path "*.app*" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz cd ../../ done mkdir -p $OUTDIR/src mv $SOURCEDIST $OUTDIR/src mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 7f93377ad..89ce10b6d 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -1,214 +1,218 @@ --- name: "bitcoin-abc-win" enable_cache: true distro: "debian" suites: - "buster" architectures: - "amd64" packages: - "autoconf" - "automake" - "bsdmainutils" - "ca-certificates" -- "cmake" - "curl" - "faketime" - "g++" - "g++-mingw-w64" - "git" - "libtool" - "ninja-build" - "mingw-w64" - "nsis" - "pkg-config" - "python3" - "zip" # FIXME: these dependencies are only required to make CMake happy when building # native tools. They can be removed when the `NativeExecutable.cmake` gets # improved to avoid requiring all the bitcoin-abc dependencies. - "libboost-all-dev" - "libevent-dev" - "libssl-dev" +repositories: +- "distribution": "buster-backports" + "source": "deb http://deb.debian.org/debian/ buster-backports main" + packages: + - "cmake" remotes: - "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git" "dir": "bitcoin" files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-w64-mingw32" # CMake toolchain file name differ from host name declare -A CMAKE_TOOLCHAIN_FILE CMAKE_TOOLCHAIN_FILE[x86_64-w64-mingw32]=Win64.cmake INSTALL_COMPONENTS="bitcoind bitcoin-qt" FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export GZIP="-9n" export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } function create_per-host_linker_wrapper { # This is only needed for trusty, as the mingw linker leaks a few bytes of # heap, causing non-determinism. See discussion in https://github.com/bitcoin/bitcoin/pull/6900 for i in $HOSTS; do mkdir -p ${WRAP_DIR}/${i} for prog in collect2; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}/${prog} REAL=$(${i}-gcc -print-prog-name=${prog}) echo "export MALLOC_PERTURB_=255" >> ${WRAP_DIR}/${i}/${prog} echo "${REAL} \$@" >> $WRAP_DIR/${i}/${prog} chmod +x ${WRAP_DIR}/${i}/${prog} done for prog in gcc g++; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" create_per-host_faketime_wrappers "2000-01-01 12:00:00" create_per-host_linker_wrapper "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} cd bitcoin SOURCEDIR=`pwd` BASEPREFIX=`pwd`/depends # Build dependencies for each host for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done # Faketime for binaries export PATH=${PATH_orig} create_global_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_linker_wrapper "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} mkdir -p source_package pushd source_package cmake -GNinja .. \ -DBUILD_BITCOIN_WALLET=OFF \ -DBUILD_BITCOIN_ZMQ=OFF \ -DBUILD_BITCOIN_SEEDER=OFF \ -DBUILD_BITCOIN_CLI=OFF \ -DBUILD_BITCOIN_TX=OFF \ -DBUILD_BITCOIN_QT=OFF \ -DBUILD_LIBBITCOINCONSENSUS=OFF \ -DENABLE_CLANG_TIDY=OFF \ -DENABLE_QRCODE=OFF \ -DENABLE_UPNP=OFF \ -DUSE_JEMALLOC=OFF ninja package_source SOURCEDIST=`echo bitcoin-abc-*.tar.gz` mv ${SOURCEDIST} .. popd DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` # Correct tar file order mkdir -p temp pushd temp tar xf ../$SOURCEDIST find bitcoin-abc-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST mkdir -p $OUTDIR/src cp ../$SOURCEDIST $OUTDIR/src popd # Allow extra cmake option to be specified for each host declare -A CMAKE_EXTRA_OPTIONS CMAKE_EXTRA_OPTIONS[x86_64-w64-mingw32]="-DCPACK_PACKAGE_FILE_NAME=${DISTNAME}-win64-setup-unsigned" ORIGPATH="$PATH" # Extract the release tarball into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${WRAP_DIR}:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH=`pwd`/installed/${DISTNAME} mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST cmake -GNinja .. \ -DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${i}]} \ -DCLIENT_VERSION_IS_RELEASE=ON \ -DENABLE_CLANG_TIDY=OFF \ -DENABLE_REDUCE_EXPORTS=ON \ -DBUILD_BITCOIN_SEEDER=OFF \ -DCPACK_STRIP_FILES=ON \ -DCMAKE_INSTALL_PREFIX=${INSTALLPATH} \ -DCCACHE=OFF \ ${CMAKE_EXTRA_OPTIONS[${i}]} ninja ninja security-check for _component in ${INSTALL_COMPONENTS}; do cmake -DCOMPONENT=${_component} -P cmake_install.cmake done ninja package cp -f bitcoin-abc-*-setup-unsigned.exe ${OUTDIR}/ cd installed mkdir -p ${DISTNAME}/lib mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ find ${DISTNAME}/bin -type f -executable -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; find ${DISTNAME}/lib -type f -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}-debug.zip cd ../../ rm -rf distsrc-${i} done cd $OUTDIR find . -name "*-setup-unsigned.exe" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz mv ${OUTDIR}/${DISTNAME}-x86_64-*-debug.zip ${OUTDIR}/${DISTNAME}-win64-debug.zip mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip