diff --git a/cmake/platforms/OSX.cmake b/cmake/platforms/OSX.cmake index 67265463a4..92878c9ac4 100644 --- a/cmake/platforms/OSX.cmake +++ b/cmake/platforms/OSX.cmake @@ -1,51 +1,45 @@ # Copyright (c) 2017 The Bitcoin developers set(CMAKE_SYSTEM_NAME Darwin) set(CMAKE_SYSTEM_PROCESSOR x86_64) set(TOOLCHAIN_PREFIX ${CMAKE_SYSTEM_PROCESSOR}-apple-darwin16) # On OSX, we use clang by default. set(CMAKE_C_COMPILER clang) set(CMAKE_CXX_COMPILER clang++) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN_PREFIX}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN_PREFIX}) # On OSX we use various stuff from Apple's SDK. -set(OSX_SDK_PATH "${CMAKE_CURRENT_SOURCE_DIR}/depends/SDKs/MacOSX10.11.sdk") +set(OSX_SDK_PATH "${CMAKE_CURRENT_SOURCE_DIR}/depends/SDKs/MacOSX10.14.sdk") set(CMAKE_OSX_SYSROOT ${OSX_SDK_PATH}) set(CMAKE_OSX_DEPLOYMENT_TARGET 10.12) set(CMAKE_OSX_ARCHITECTURES x86_64) # target environment on the build host system # set 1st to dir with the cross compiler's C/C++ headers/libs set(CMAKE_FIND_ROOT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/depends/${TOOLCHAIN_PREFIX};${OSX_SDK_PATH}") # We also may have built dependencies for the native plateform. set(CMAKE_PREFIX_PATH "${CMAKE_CURRENT_SOURCE_DIR}/depends/${TOOLCHAIN_PREFIX}/native") # modify default behavior of FIND_XXX() commands to # search for headers/libs in the target environment and # search for programs in the build host environment set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -# Sysroot clang -set(OSX_EXTRA_FLAGS - " -mlinker-version=253.9" -) - -string(APPEND CMAKE_C_FLAGS_INIT ${OSX_EXTRA_FLAGS}) -string(APPEND CMAKE_CXX_FLAGS_INIT ${OSX_EXTRA_FLAGS} " -stdlib=libc++") +string(APPEND CMAKE_CXX_FLAGS_INIT " -stdlib=libc++") # Ensure we use an OSX specific version the binary manipulation tools. find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}-ar) find_program(CMAKE_INSTALL_NAME_TOOL ${TOOLCHAIN_PREFIX}-install_name_tool) find_program(CMAKE_LINKER ${TOOLCHAIN_PREFIX}-ld) find_program(CMAKE_NM ${TOOLCHAIN_PREFIX}-nm) find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}-objcopy) find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}-objdump) find_program(CMAKE_OTOOL ${TOOLCHAIN_PREFIX}-otool) find_program(CMAKE_RANLIB ${TOOLCHAIN_PREFIX}-ranlib) find_program(CMAKE_STRIP ${TOOLCHAIN_PREFIX}-strip) diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index 4b95b0a17d..33057fb25d 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -1,314 +1,314 @@ #!/usr/bin/env python3 import argparse import multiprocessing import os import subprocess import sys def setup(): global args, workdir programs = ['ruby', 'git', 'apt-cacher-ng', 'make', 'wget'] if args.kvm: programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils'] elif args.docker: dockers = ['docker.io', 'docker-ce'] for i in dockers: return_code = subprocess.call( ['sudo', 'apt-get', 'install', '-qq', i]) if return_code == 0: break if return_code != 0: print('Cannot find any way to install docker', file=sys.stderr) exit(1) else: programs += ['lxc', 'debootstrap'] subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) if not os.path.isdir('gitian-builder'): subprocess.check_call( ['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) if not os.path.isdir('bitcoin-abc'): subprocess.check_call( ['git', 'clone', 'https://github.com/Bitcoin-ABC/bitcoin-abc.git']) os.chdir('gitian-builder') make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] if args.docker: make_image_prog += ['--docker'] elif not args.kvm: make_image_prog += ['--lxc'] subprocess.check_call(make_image_prog) os.chdir(workdir) if args.is_bionic and not args.kvm and not args.docker: subprocess.check_call( ['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) print('Reboot is required') exit(0) def build(): global args, workdir base_output_dir = 'bitcoin-binaries/' + args.version os.makedirs(base_output_dir + '/src', exist_ok=True) print('\nBuilding Dependencies\n') os.chdir('gitian-builder') os.makedirs('inputs', exist_ok=True) subprocess.check_call(['make', '-C', '../bitcoin-abc/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) output_dir_src = '../' + base_output_dir + '/src' if args.linux: print('\nCompiling ' + args.version + ' Linux') subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin=' + args.commit, '--url', 'bitcoin=' + args.url, '../bitcoin-abc/contrib/gitian-descriptors/gitian-linux.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version + '-linux', '--destination', '../gitian.sigs/', '../bitcoin-abc/contrib/gitian-descriptors/gitian-linux.yml']) output_dir_linux = '../' + base_output_dir + '/linux' os.makedirs(output_dir_linux, exist_ok=True) subprocess.check_call( 'mv build/out/bitcoin-*.tar.gz ' + output_dir_linux, shell=True) subprocess.check_call( 'mv build/out/src/bitcoin-*.tar.gz ' + output_dir_src, shell=True) subprocess.check_call( 'mv result/bitcoin-*-linux-res.yml ' + output_dir_linux, shell=True) if args.windows: print('\nCompiling ' + args.version + ' Windows') subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin=' + args.commit, '--url', 'bitcoin=' + args.url, '../bitcoin-abc/contrib/gitian-descriptors/gitian-win.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version + '-win-unsigned', '--destination', '../gitian.sigs/', '../bitcoin-abc/contrib/gitian-descriptors/gitian-win.yml']) output_dir_win = '../' + base_output_dir + '/win' os.makedirs(output_dir_win, exist_ok=True) subprocess.check_call( 'mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/', shell=True) subprocess.check_call( 'mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe ' + output_dir_win, shell=True) subprocess.check_call( 'mv build/out/src/bitcoin-*.tar.gz ' + output_dir_src, shell=True) subprocess.check_call( 'mv result/bitcoin-*-win-res.yml ' + output_dir_win, shell=True) if args.macos: print('\nCompiling ' + args.version + ' MacOS') subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin=' + args.commit, '--url', 'bitcoin=' + args.url, '../bitcoin-abc/contrib/gitian-descriptors/gitian-osx.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version + '-osx-unsigned', '--destination', '../gitian.sigs/', '../bitcoin-abc/contrib/gitian-descriptors/gitian-osx.yml']) output_dir_osx = '../' + base_output_dir + '/osx' os.makedirs(output_dir_osx, exist_ok=True) subprocess.check_call( 'mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/', shell=True) subprocess.check_call( 'mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ' + output_dir_osx, shell=True) subprocess.check_call( 'mv build/out/src/bitcoin-*.tar.gz ' + output_dir_src, shell=True) subprocess.check_call( 'mv result/bitcoin-*-osx-res.yml ' + output_dir_osx, shell=True) os.chdir(workdir) if args.commit_files: print('\nCommitting ' + args.version + ' Unsigned Sigs\n') os.chdir('gitian.sigs') subprocess.check_call( ['git', 'add', args.version + '-linux/' + args.signer]) subprocess.check_call( ['git', 'add', args.version + '-win-unsigned/' + args.signer]) subprocess.check_call( ['git', 'add', args.version + '-osx-unsigned/' + args.signer]) subprocess.check_call( ['git', 'commit', '-m', 'Add ' + args.version + ' unsigned sigs for ' + args.signer]) os.chdir(workdir) def sign(): global args, workdir os.chdir('gitian-builder') if args.windows: print('\nSigning ' + args.version + ' Windows') subprocess.check_call('cp inputs/bitcoin-' + args.version + '-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz', shell=True) subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature=' + args.commit, '../bitcoin-abc/contrib/gitian-descriptors/gitian-win-signer.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version + '-win-signed', '--destination', '../gitian.sigs/', '../bitcoin-abc/contrib/gitian-descriptors/gitian-win-signer.yml']) subprocess.check_call( 'mv build/out/bitcoin-*win64-setup.exe ../bitcoin-binaries/' + args.version, shell=True) subprocess.check_call( 'mv build/out/bitcoin-*win32-setup.exe ../bitcoin-binaries/' + args.version, shell=True) if args.macos: print('\nSigning ' + args.version + ' MacOS') subprocess.check_call('cp inputs/bitcoin-' + args.version + '-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz', shell=True) subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature=' + args.commit, '../bitcoin-abc/contrib/gitian-descriptors/gitian-osx-signer.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version + '-osx-signed', '--destination', '../gitian.sigs/', '../bitcoin-abc/contrib/gitian-descriptors/gitian-osx-signer.yml']) subprocess.check_call('mv build/out/bitcoin-osx-signed.dmg ../bitcoin-binaries/' + args.version + '/bitcoin-' + args.version + '-osx.dmg', shell=True) os.chdir(workdir) if args.commit_files: print('\nCommitting ' + args.version + ' Signed Sigs\n') os.chdir('gitian.sigs') subprocess.check_call( ['git', 'add', args.version + '-win-signed/' + args.signer]) subprocess.check_call( ['git', 'add', args.version + '-osx-signed/' + args.signer]) subprocess.check_call(['git', 'commit', '-a', '-m', 'Add ' + args.version + ' signed binary sigs for ' + args.signer]) os.chdir(workdir) def verify(): global args, workdir os.chdir('gitian-builder') print('\nVerifying v' + args.version + ' Linux\n') subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version + '-linux', '../bitcoin-abc/contrib/gitian-descriptors/gitian-linux.yml']) print('\nVerifying v' + args.version + ' Windows\n') subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version + '-win-unsigned', '../bitcoin-abc/contrib/gitian-descriptors/gitian-win.yml']) print('\nVerifying v' + args.version + ' MacOS\n') subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version + '-osx-unsigned', '../bitcoin-abc/contrib/gitian-descriptors/gitian-osx.yml']) print('\nVerifying v' + args.version + ' Signed Windows\n') subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version + '-win-signed', '../bitcoin-abc/contrib/gitian-descriptors/gitian-win-signer.yml']) print('\nVerifying v' + args.version + ' Signed MacOS\n') subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version + '-osx-signed', '../bitcoin-abc/contrib/gitian-descriptors/gitian-osx-signer.yml']) os.chdir(workdir) def main(): global args, workdir num_cpus = multiprocessing.cpu_count() parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version') parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') parser.add_argument('-u', '--url', dest='url', default='https://github.com/Bitcoin-ABC/bitcoin-abc.git', help='Specify the URL of the repository. Default is %(default)s') parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') parser.add_argument('-s', '--sign', action='store_true', dest='sign', help='Make signed binaries for Windows and MacOS') parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') parser.add_argument('-j', '--jobs', dest='jobs', default=str(num_cpus), help='Number of processes to use. Default %(default)s') parser.add_argument('-m', '--memory', dest='memory', default='3500', help='Memory to allocate in MiB. Default %(default)s') parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian)') parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') parser.add_argument( 'signer', help='GPG signer to sign each build assert file') parser.add_argument( 'version', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') args = parser.parse_args() workdir = os.getcwd() args.linux = 'l' in args.os args.windows = 'w' in args.os args.macos = 'm' in args.os args.is_bionic = b'bionic' in subprocess.check_output( ['lsb_release', '-cs']) if args.buildsign: args.build = True args.sign = True if args.kvm and args.docker: raise Exception('Error: cannot have both kvm and docker') args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' # Set environment variable USE_LXC or USE_DOCKER, let gitian-builder know # that we use lxc or docker if args.docker: os.environ['USE_DOCKER'] = '1' elif not args.kvm: os.environ['USE_LXC'] = '1' if 'GITIAN_HOST_IP' not in os.environ.keys(): os.environ['GITIAN_HOST_IP'] = '10.0.3.1' if 'LXC_GUEST_IP' not in os.environ.keys(): os.environ['LXC_GUEST_IP'] = '10.0.3.5' # Disable for MacOS if no SDK found if args.macos and not os.path.isfile( - 'gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): + 'gitian-builder/inputs/MacOSX10.14.sdk.tar.gz'): print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') args.macos = False script_name = os.path.basename(sys.argv[0]) # Signer and version shouldn't be empty if args.signer == '': print(script_name + ': Missing signer.') print('Try ' + script_name + ' --help for more information') exit(1) if args.version == '': print(script_name + ': Missing version.') print('Try ' + script_name + ' --help for more information') exit(1) # Add leading 'v' for tags if args.commit and args.pull: raise Exception('Cannot have both commit and pull') args.commit = ('' if args.commit else 'v') + args.version if args.setup: setup() os.chdir('bitcoin-abc') if args.pull: subprocess.check_call( ['git', 'fetch', args.url, 'refs/pull/' + args.version + '/merge']) os.chdir('../gitian-builder/inputs/bitcoin') subprocess.check_call( ['git', 'fetch', args.url, 'refs/pull/' + args.version + '/merge']) args.commit = subprocess.check_output( ['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True, encoding='utf8').strip() args.version = 'pull-' + args.version print(args.commit) subprocess.check_call(['git', 'fetch']) subprocess.check_call(['git', 'checkout', args.commit]) os.chdir(workdir) if args.build: build() if args.sign: sign() if args.verify: verify() if __name__ == '__main__': main() diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 74d63b65e8..04354fba1a 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,205 +1,205 @@ --- name: "bitcoin-abc-osx" enable_cache: true distro: "debian" suites: - "buster" architectures: - "amd64" packages: - "autoconf" - "automake" - "bsdmainutils" - "ca-certificates" - "cmake" - "curl" - "faketime" - "fonts-tuffy" - "g++" - "git" - "imagemagick" - "libbz2-dev" - "libcap-dev" - "librsvg2-bin" - "libtiff-tools" - "libtinfo5" - "libtool" - "libz-dev" - "ninja-build" - "pkg-config" - "python3" - "python3-dev" - "python3-setuptools" # FIXME: these dependencies are only required to make CMake happy when building # native tools. They can be removed when the `NativeExecutable.cmake` gets # improved to avoid requiring all the bitcoin-abc dependencies. - "libboost-all-dev" - "libevent-dev" - "libssl-dev" remotes: - "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git" "dir": "bitcoin" files: -- "MacOSX10.11.sdk.tar.gz" +- "MacOSX10.14.sdk.tar.gz" script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin16" # CMake toolchain file name differ from host name declare -A CMAKE_TOOLCHAIN_FILE CMAKE_TOOLCHAIN_FILE[x86_64-apple-darwin16]=OSX.cmake INSTALL_COMPONENTS="bitcoind bitcoin-qt bitcoin-seeder" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="ar ranlib date dmg genisoimage" export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export GZIP="-9n" export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi export ZERO_AR_DATE=1 function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" create_per-host_faketime_wrappers "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} cd bitcoin SOURCEDIR=`pwd` BASEPREFIX=`pwd`/depends mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz # Build dependencies for each host for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done # Faketime for binaries export PATH=${PATH_orig} create_global_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} mkdir -p source_package pushd source_package cmake -GNinja .. \ -DBUILD_BITCOIN_WALLET=OFF \ -DBUILD_BITCOIN_ZMQ=OFF \ -DBUILD_BITCOIN_ZMQ=OFF \ -DBUILD_BITCOIN_SEEDER=OFF \ -DBUILD_BITCOIN_CLI=OFF \ -DBUILD_BITCOIN_TX=OFF \ -DBUILD_BITCOIN_QT=OFF \ -DBUILD_LIBBITCOINCONSENSUS=OFF \ -DENABLE_CLANG_TIDY=OFF \ -DENABLE_QRCODE=OFF \ -DENABLE_UPNP=OFF ninja package_source SOURCEDIST=`echo bitcoin-abc-*.tar.gz` mv ${SOURCEDIST} .. popd DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` # Correct tar file order mkdir -p temp pushd temp tar xf ../$SOURCEDIST find bitcoin-abc-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST popd ORIGPATH="$PATH" # Extract the release tarball into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH=`pwd`/installed/${DISTNAME} mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST cmake -GNinja .. \ -DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${i}]} \ -DCLIENT_VERSION_IS_RELEASE=ON \ -DENABLE_CLANG_TIDY=OFF \ -DENABLE_REDUCE_EXPORTS=ON \ -DCMAKE_INSTALL_PREFIX=${INSTALLPATH} \ -DCCACHE=OFF \ -DGENISOIMAGE_EXECUTABLE="${WRAP_DIR}/genisoimage" ninja for _component in ${INSTALL_COMPONENTS}; do cmake -DCOMPONENT=${_component} -P cmake_install.cmake done export PYTHONPATH="${BASEPREFIX}/${i}/native/lib/python3/dist-packages:${PYTHONPATH}" ninja osx-deploydir OSX_VOLNAME="$(cat osx_volname)" mkdir -p unsigned-app-${i} cp osx_volname unsigned-app-${i}/ cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i} cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff mv dist unsigned-app-${i} pushd unsigned-app-${i} find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz popd ninja osx-dmg ${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg cd installed # The install/strip target should do exactly that. However there is a bug in # in the Buster CMake version that causes strip to be called with the wrong # parameters when an Apple strip tool is used. This is fixed starting with # CMake versions >= 3.14, see: # https://gitlab.kitware.com/cmake/cmake/merge_requests/2892 find -path "*.app*" -type f -executable -exec mv {} ${DISTNAME}/bin/bitcoin-qt \; find ${DISTNAME}/bin -type f -executable -exec ${i}-strip -u -r {} \; find ${DISTNAME}/lib -type f -executable -exec ${i}-strip -x {} \; find ${DISTNAME} -not -path "*.app*" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz cd ../../ done mkdir -p $OUTDIR/src mv $SOURCEDIST $OUTDIR/src mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 6163734e62..505e86e171 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -1,15 +1,135 @@ -### MacDeploy ### +# MacOS Deployment -For Snow Leopard (which uses [Python 2.6](http://www.python.org/download/releases/2.6/)), you will need the param_parser package: - - sudo easy_install argparse +The `macdeployqtplus` script should not be run manually. Instead, after building as usual: -This script should not be run manually, instead, after building as usual: +```bash +ninja osx-dmg +``` - make deploy +During the deployment process, the disk image window will pop up briefly +when the fancy settings are applied. This is normal, please do not interfere, +the process will unmount the DMG and cleanup before finishing. -During the process, the disk image window will pop up briefly where the fancy -settings are applied. This is normal, please do not interfere. +When complete, it will have produced `Bitcoin-ABC.dmg`. -When finished, it will produce `Bitcoin-Core.dmg`. +## SDK Extraction +`Xcode.app` is packaged in a `.xip` archive. +This makes the SDK less-trivial to extract on non-macOS machines. +One approach (tested on Debian Buster) is outlined below: + +```bash + +apt install clang cpio git liblzma-dev libxml2-dev libssl-dev make + +git clone https://github.com/tpoechtrager/xar +pushd xar/xar +./configure +make +make install +popd + +git clone https://github.com/NiklasRosenstein/pbzx +pushd pbzx +clang -llzma -lxar pbzx.c -o pbzx -Wl,-rpath=/usr/local/lib +popd + +xar -xf Xcode_10.2.1.xip -C . + +./pbzx/pbzx -n Content | cpio -i + +find Xcode.app -type d -name MacOSX.sdk -execdir sh -c 'tar -c MacOSX.sdk/ | gzip -9n > /MacOSX10.14.sdk.tar.gz' \; +``` + +on macOS the process is more straightforward: + +```bash +xip -x Xcode_10.2.1.xip +tar -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk +``` + +Our previously used macOS SDK (`MacOSX10.11.sdk`) can be extracted from +[Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). +The script [`extract-osx-sdk.sh`](./extract-osx-sdk.sh) automates this. First +ensure the DMG file is in the current directory, and then run the script. You +may wish to delete the `intermediate 5.hfs` file and `MacOSX10.11.sdk` (the +directory) when you've confirmed the extraction succeeded. + +```bash +apt-get install p7zip-full sleuthkit +contrib/macdeploy/extract-osx-sdk.sh +rm -rf 5.hfs MacOSX10.11.sdk +``` + +## Deterministic macOS DMG Notes +Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple +`binutils` (`ld`, `ar`, etc) and DMG authoring tools. + +Apple uses `clang` extensively for development and has upstreamed the necessary +functionality so that a vanilla clang can take advantage. It supports the use of `-F`, +`-target`, `-mmacosx-version-min`, and `--sysroot`, which are all necessary when +building for macOS. + +Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the +FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several +other tools are needed as well such as `install_name_tool`, `lipo`, and `nmedit`. These +do not build under Linux, so they have been patched to do so. The work here was used as +a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). + +In order to build a working toolchain, the following source packages are needed from +Apple: `cctools`, `dyld`, and `ld64`. + +These tools inject timestamps by default, which produce non-deterministic binaries. The +`ZERO_AR_DATE` environment variable is used to disable that. + +This version of `cctools` has been patched to use the current version of `clang`'s headers +and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`. + +To complicate things further, all builds must target an Apple SDK. These SDKs are free to +download, but not redistributable. To obtain it, register for an Apple Developer Account, +then download [Xcode 10.2.1](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). + +This file is many gigabytes in size, but most (but not all) of what we need is +contained only in a single directory: + +```bash +Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk +``` + +See the SDK Extraction notes above for how to obtain it. + +The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries which are +created using these tools. The build process has been designed to avoid including the +SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely +redistributed. + +`genisoimage` is used to create the initial DMG. It is not deterministic as-is, so it has been +patched. A system `genisoimage` will work fine, but it will not be deterministic because +the file-order will change between invocations. The patch can be seen here: [cdrkit-deterministic.patch](https://github.com/bitcoin/bitcoin/blob/master/depends/patches/native_cdrkit/cdrkit-deterministic.patch). +No effort was made to fix this cleanly, so it likely leaks memory badly, however it's only used for +a single invocation, so that's no real concern. + +`genisoimage` cannot compress DMGs, so afterwards, the DMG tool from the +`libdmg-hfsplus` project is used to compress it. There are several bugs in this tool and its +maintainer has seemingly abandoned the project. + +The DMG tool has the ability to create DMGs from scratch as well, but this functionality is +broken. Only the compression feature is currently used. Ideally, the creation could be fixed +and `genisoimage` would no longer be necessary. + +Background images and other features can be added to DMG files by inserting a +`.DS_Store` before creation. This is generated by the script `contrib/macdeploy/custom_dsstore.py`. + +As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in +order to satisfy the new Gatekeeper requirements. Because this private key cannot be +shared, we'll have to be a bit creative in order for the build process to remain somewhat +deterministic. Here's how it works: + +- Builders use Gitian to create an unsigned release. This outputs an unsigned DMG which + users may choose to bless and run. It also outputs an unsigned app structure in the form + of a tarball, which also contains all of the tools that have been previously (deterministically) + built in order to create a final DMG. +- The Apple keyholder uses this unsigned app to create a detached signature, using the + script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). +- Builders feed the unsigned app + detached signature back into Gitian. It uses the + pre-built tools to recombine the pieces into a deterministic DMG. diff --git a/contrib/teamcity/gitian.sh b/contrib/teamcity/gitian.sh index 4c9d4c4755..0e59563bfd 100755 --- a/contrib/teamcity/gitian.sh +++ b/contrib/teamcity/gitian.sh @@ -1,50 +1,50 @@ #!/usr/bin/env bash export LC_ALL=C set -euxo pipefail cd "$(dirname "$0")" COMMIT=$(git rev-parse HEAD) export COMMIT PROJECT_ROOT=$(git rev-parse --show-toplevel) export PROJECT_ROOT export USE_LXC=1 export GITIAN_HOST_IP=10.0.3.1 export LXC_BRIDGE=lxcbr0 export LXC_GUEST_IP=10.0.3.5 cd ~/gitian-builder if [[ "${OS_NAME}" == "osx" ]]; then - OSX_SDK="MacOSX10.11.sdk.tar.gz" - OSX_SDK_SHA256="4732b52b5ebe300c8c91cbeed6d19d59c1ff9c56c7a1dd6cfa518b9c2c72abde" + OSX_SDK="MacOSX10.14.sdk.tar.gz" + OSX_SDK_SHA256="2322086a96349db832abbcadea493b79db843553a2e604163238d99fa058a286" mkdir -p inputs pushd inputs if ! echo "${OSX_SDK_SHA256} ${OSX_SDK}" | sha256sum -c; then rm -f "${OSX_SDK}" - wget https://storage.googleapis.com/f4936e83b2dcbca742be51fb9692b153/"${OSX_SDK}" + wget https://storage.googleapis.com/27cd7b2a42a430926cc621acdc3bda72a8ed2b0efc080e3/"${OSX_SDK}" echo "${OSX_SDK_SHA256} ${OSX_SDK}" | sha256sum -c fi popd fi ## Determine the number of build threads THREADS=$(nproc || sysctl -n hw.ncpu) RESULT_DIR="${PROJECT_ROOT}/gitian-results" OS_DIR="${RESULT_DIR}/${OS_NAME}" mkdir -p "${OS_DIR}" move_log() { mv var/install.log "${RESULT_DIR}/" mv var/build.log "${RESULT_DIR}/" } trap "move_log" ERR ./bin/gbuild -j${THREADS} -m3500 --commit bitcoin=${COMMIT} --url bitcoin="${PROJECT_ROOT}" "${PROJECT_ROOT}/contrib/gitian-descriptors/gitian-${OS_NAME}.yml" move_log mv result/*.yml "${OS_DIR}/" mv build/out/* "${OS_DIR}/" diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 1f88c209cf..1bc4fb8189 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,17 +1,16 @@ OSX_MIN_VERSION=10.12 -OSX_SDK_VERSION=10.11 +OSX_SDK_VERSION=10.14 OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk -LD64_VERSION=253.9 -darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -stdlib=libc++ +darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) +darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) darwin_release_CFLAGS=-O2 darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) darwin_debug_CFLAGS=-O1 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) darwin_native_toolchain=native_cctools diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 9fa4549f41..416c903a07 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -1,65 +1,78 @@ package=native_cctools -$(package)_version=807d6fd1be5d2224872e381870c0a75387fe05e6 -$(package)_download_path=https://github.com/theuni/cctools-port/archive +$(package)_version=3764b223c011574971ee3ae09ce968ba5dc2f00f +$(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=a09c9ba4684670a0375e42d9d67e7f12c1f62581a27f28f7c825d6d7032ccc6a +$(package)_sha256_hash=3e35907bf376269a844df08e03cbb43e345c88125374f2228e03724b5f9a2a04 $(package)_build_subdir=cctools -$(package)_clang_version=5.0.1 -$(package)_clang_download_path=https://llvm.org/releases/$($(package)_clang_version) +$(package)_clang_version=6.0.1 +$(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version) $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz -$(package)_clang_sha256_hash=005f21861cd2953138df7cf511f1552ef7041bafb7cfc4b172264b7ff5fe09b4 +$(package)_clang_sha256_hash=7ea204ecd78c39154d72dfc0d4a79f7cce1b2264da2551bb2eef10e266d54d91 + +$(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef +$(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive +$(package)_libtapi_download_file=$($(package)_libtapi_version).tar.gz +$(package)_libtapi_file_name=$($(package)_libtapi_version).tar.gz +$(package)_libtapi_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3 + $(package)_extra_sources=$($(package)_clang_file_name) +$(package)_extra_sources += $($(package)_libtapi_file_name) define $(package)_fetch_cmds $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) +$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash)) endef define $(package)_extract_cmds mkdir -p $($(package)_extract_dir) && \ echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ echo "$($(package)_clang_sha256_hash) $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_libtapi_sha256_hash) $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - mkdir -p toolchain/bin && \ + mkdir -p toolchain/bin toolchain/lib/clang/$($(package)_clang_version)/include && \ + mkdir -p libtapi && \ + tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \ tar --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \ rm -f toolchain/lib/libc++abi.so* && \ - echo "#!/bin/sh" > toolchain/bin/$(host)-dsymutil && \ - echo "exit 0" >> toolchain/bin/$(host)-dsymutil && \ - chmod +x toolchain/bin/$(host)-dsymutil && \ tar --strip-components=1 -xf $($(package)_source) endef define $(package)_set_vars -$(package)_config_opts=--target=$(host) --disable-lto-support -$(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib -$(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang -$(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++ + $(package)_config_opts=--target=$(host) --disable-lto-support --with-libtapi=$($(package)_extract_dir) + $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib + $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang + $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++ endef define $(package)_preprocess_cmds - cd $($(package)_build_subdir); ./autogen.sh && \ - sed -i.old "/define HAVE_PTHREADS/d" ld64/src/ld/InputFiles.h + CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/build.sh && \ + CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/install.sh && \ + sed -i.old "/define HAVE_PTHREADS/d" $($(package)_build_subdir)/ld64/src/ld/InputFiles.h endef define $(package)_config_cmds $($(package)_autoconf) endef define $(package)_build_cmds $(MAKE) -j$(JOBS) endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install && \ + mkdir -p $($(package)_staging_prefix_dir)/lib/ && \ + cd $($(package)_extract_dir) && \ + cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/ && \ cd $($(package)_extract_dir)/toolchain && \ mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include && \ mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \ cp bin/clang $($(package)_staging_prefix_dir)/bin/ &&\ cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\ cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \ cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \ if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi endef diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 5a9adab836..914b457e40 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,189 +1,188 @@ PACKAGE=qt $(package)_version=5.9.6 $(package)_download_path=https://download.qt.io/official_releases/qt/5.9/$($(package)_version)/submodules $(package)_suffix=opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) $(package)_sha256_hash=eed620cb268b199bd83b3fc6a471c51d51e1dc2dbb5374fc97a0cc75facbe36f $(package)_dependencies=openssl zlib $(package)_linux_dependencies=freetype fontconfig libxcb libX11 xproto libXext $(package)_build_subdir=qtbase $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch xkb-default.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) $(package)_qttranslations_sha256_hash=9822084f8e2d2939ba39f4af4c0c2320e45d5996762a9423f833055607604ed8 $(package)_qttools_file_name=qttools-$($(package)_suffix) $(package)_qttools_sha256_hash=50e75417ec0c74bb8b1989d1d8e981ee83690dce7dfc0c2169f7c00f397e5117 $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) define $(package)_set_vars $(package)_config_opts_release = -release $(package)_config_opts_debug = -debug $(package)_config_opts += -bindir $(build_prefix)/bin $(package)_config_opts += -c++std c++11 $(package)_config_opts += -confirm-license $(package)_config_opts += -dbus-runtime $(package)_config_opts += -hostprefix $(build_prefix) $(package)_config_opts += -no-cups $(package)_config_opts += -no-egl $(package)_config_opts += -no-eglfs $(package)_config_opts += -no-freetype $(package)_config_opts += -no-gif $(package)_config_opts += -no-glib $(package)_config_opts += -no-icu $(package)_config_opts += -no-iconv $(package)_config_opts += -no-kms $(package)_config_opts += -no-linuxfb $(package)_config_opts += -no-libudev $(package)_config_opts += -no-mtdev $(package)_config_opts += -no-opengl $(package)_config_opts += -no-openvg $(package)_config_opts += -no-reduce-relocations $(package)_config_opts += -no-qml-debug $(package)_config_opts += -no-sql-db2 $(package)_config_opts += -no-sql-ibase $(package)_config_opts += -no-sql-oci $(package)_config_opts += -no-sql-tds $(package)_config_opts += -no-sql-mysql $(package)_config_opts += -no-sql-odbc $(package)_config_opts += -no-sql-psql $(package)_config_opts += -no-sql-sqlite $(package)_config_opts += -no-sql-sqlite2 $(package)_config_opts += -no-use-gold-linker $(package)_config_opts += -no-xinput2 $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests $(package)_config_opts += -opensource $(package)_config_opts += -openssl-linked $(package)_config_opts += -optimized-qmake $(package)_config_opts += -pch $(package)_config_opts += -pkg-config $(package)_config_opts += -prefix $(host_prefix) $(package)_config_opts += -qt-libpng $(package)_config_opts += -qt-libjpeg $(package)_config_opts += -qt-pcre $(package)_config_opts += -qt-harfbuzz $(package)_config_opts += -system-zlib $(package)_config_opts += -static $(package)_config_opts += -silent $(package)_config_opts += -v $(package)_config_opts += -no-feature-printer $(package)_config_opts += -no-feature-printdialog $(package)_config_opts += -no-feature-concurrent $(package)_config_opts += -no-feature-xml ifneq ($(build_os),darwin) $(package)_config_opts_darwin = -xplatform macx-clang-linux $(package)_config_opts_darwin += -device-option MAC_SDK_PATH=$(OSX_SDK) $(package)_config_opts_darwin += -device-option MAC_SDK_VERSION=$(OSX_SDK_VERSION) $(package)_config_opts_darwin += -device-option CROSS_COMPILE="$(host)-" $(package)_config_opts_darwin += -device-option MAC_MIN_VERSION=$(OSX_MIN_VERSION) $(package)_config_opts_darwin += -device-option MAC_TARGET=$(host) -$(package)_config_opts_darwin += -device-option MAC_LD64_VERSION=$(LD64_VERSION) endif $(package)_config_opts_linux = -qt-xkbcommon-x11 $(package)_config_opts_linux += -qt-xcb $(package)_config_opts_linux += -system-freetype $(package)_config_opts_linux += -no-feature-sessionmanager $(package)_config_opts_linux += -fontconfig $(package)_config_opts_arm_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ $(package)_config_opts_i686_linux = -xplatform linux-g++-32 $(package)_config_opts_x86_64_linux = -xplatform linux-g++-64 $(package)_config_opts_aarch64_linux = -xplatform linux-aarch64-gnu-g++ $(package)_config_opts_mingw32 = -xplatform win32-g++ -device-option CROSS_COMPILE="$(host)-" $(package)_build_env = QT_RCC_TEST=1 $(package)_build_env += QT_RCC_SOURCE_DATE_OVERRIDE=1 endef define $(package)_fetch_cmds $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ $(call fetch_file,$(package),$($(package)_download_path),$($(package)_qttranslations_file_name),$($(package)_qttranslations_file_name),$($(package)_qttranslations_sha256_hash)) && \ $(call fetch_file,$(package),$($(package)_download_path),$($(package)_qttools_file_name),$($(package)_qttools_file_name),$($(package)_qttools_sha256_hash)) endef define $(package)_extract_cmds mkdir -p $($(package)_extract_dir) && \ echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ echo "$($(package)_qttranslations_sha256_hash) $($(package)_source_dir)/$($(package)_qttranslations_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ echo "$($(package)_qttools_sha256_hash) $($(package)_source_dir)/$($(package)_qttools_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ mkdir qtbase && \ tar --strip-components=1 -xf $($(package)_source) -C qtbase && \ mkdir qttranslations && \ tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \ mkdir qttools && \ tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools endef define $(package)_preprocess_cmds sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \ sed -i.old "s/src_plugins.depends = src_sql src_network/src_plugins.depends = src_network/" qtbase/src/src.pro && \ sed -i.old "s|X11/extensions/XIproto.h|X11/X.h|" qtbase/src/plugins/platforms/xcb/qxcbxsettings.cpp && \ sed -i.old 's/if \[ "$$$$XPLATFORM_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/if \[ "$$$$BUILD_ON_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/' qtbase/configure && \ sed -i.old 's/CGEventCreateMouseEvent(0, kCGEventMouseMoved, pos, 0)/CGEventCreateMouseEvent(0, kCGEventMouseMoved, pos, kCGMouseButtonLeft)/' qtbase/src/plugins/platforms/cocoa/qcocoacursor.mm && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/Info.plist.lib qtbase/mkspecs/macx-clang-linux/ &&\ cp -f qtbase/mkspecs/macx-clang/Info.plist.app qtbase/mkspecs/macx-clang-linux/ &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ cp -r qtbase/mkspecs/linux-arm-gnueabi-g++ qtbase/mkspecs/bitcoin-linux-g++ && \ sed -i.old "s/arm-linux-gnueabi-/$(host)-/g" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch &&\ patch -p1 -i $($(package)_patch_dir)/fix_configure_mac.patch &&\ patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch &&\ patch -p1 -i $($(package)_patch_dir)/fix_rcc_determinism.patch &&\ patch -p1 -i $($(package)_patch_dir)/xkb-default.patch &&\ echo "!host_build: QMAKE_CFLAGS += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "QMAKE_LINK_OBJECT_MAX = 10" >> qtbase/mkspecs/win32-g++/qmake.conf &&\ echo "QMAKE_LINK_OBJECT_SCRIPT = object_script" >> qtbase/mkspecs/win32-g++/qmake.conf &&\ sed -i.old "s|QMAKE_CFLAGS = |!host_build: QMAKE_CFLAGS = $($(package)_cflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ sed -i.old "s|QMAKE_LFLAGS = |!host_build: QMAKE_LFLAGS = $($(package)_ldflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ sed -i.old "s|QMAKE_CXXFLAGS = |!host_build: QMAKE_CXXFLAGS = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf endef define $(package)_config_cmds export PKG_CONFIG_SYSROOT_DIR=/ && \ export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ ./configure $($(package)_config_opts) && \ echo "host_build: QT_CONFIG ~= s/system-zlib/zlib" >> mkspecs/qconfig.pri && \ echo "CONFIG += force_bootstrap" >> mkspecs/qconfig.pri && \ $(MAKE) sub-src-clean && \ cd ../qttranslations && ../qtbase/bin/qmake qttranslations.pro -o Makefile && \ cd translations && ../../qtbase/bin/qmake translations.pro -o Makefile && cd ../.. && \ cd qttools/src/linguist && ../../../qtbase/bin/qmake linguist.pro -o Makefile && \ cd lconvert && ../../../../qtbase/bin/qmake lconvert.pro -o Makefile && cd .. && \ cd lrelease && ../../../../qtbase/bin/qmake lrelease.pro -o Makefile && cd .. && \ cd lupdate && ../../../../qtbase/bin/qmake lupdate.pro -o Makefile && cd ../../../.. endef define $(package)_build_cmds $(MAKE) -j$(JOBS) && \ $(MAKE) -j$(JOBS) -C ../qttools/src/linguist/lconvert && \ $(MAKE) -j$(JOBS) -C ../qttools/src/linguist/lrelease && \ $(MAKE) -j$(JOBS) -C ../qttools/src/linguist/lupdate && \ $(MAKE) -j$(JOBS) -C ../qttranslations endef define $(package)_stage_cmds $(MAKE) INSTALL_ROOT=$($(package)_staging_dir) install && cd .. && \ $(MAKE) -C qttools/src/linguist/lconvert INSTALL_ROOT=$($(package)_staging_dir) install && \ $(MAKE) -C qttools/src/linguist/lrelease INSTALL_ROOT=$($(package)_staging_dir) install && \ $(MAKE) -C qttools/src/linguist/lupdate INSTALL_ROOT=$($(package)_staging_dir) install && \ $(MAKE) -C qttools/src/linguist INSTALL_ROOT=$($(package)_staging_dir) install_cmake_linguist_tools_files && \ $(MAKE) -C qttranslations INSTALL_ROOT=$($(package)_staging_dir) install_subtargets && \ if `test -f qtbase/src/plugins/platforms/xcb/xcb-static/libxcb-static.a`; then \ cp qtbase/src/plugins/platforms/xcb/xcb-static/libxcb-static.a $($(package)_staging_prefix_dir)/lib; \ fi endef define $(package)_postprocess_cmds rm -rf native/lib/ lib/lib*.la endef diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index 337d0eb9ca..4cd96df29f 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -1,26 +1,26 @@ MAKEFILE_GENERATOR = UNIX CONFIG += app_bundle incremental global_init_link_order lib_version_first plugin_no_soname absolute_library_soname QMAKE_INCREMENTAL_STYLE = sublib include(../common/macx.conf) include(../common/gcc-base-mac.conf) include(../common/clang.conf) include(../common/clang-mac.conf) QMAKE_MAC_SDK_PATH=$${MAC_SDK_PATH} QMAKE_XCODE_VERSION=4.3 QMAKE_XCODE_DEVELOPER_PATH=/Developer QMAKE_MACOSX_DEPLOYMENT_TARGET = $${MAC_MIN_VERSION} QMAKE_MAC_SDK=macosx QMAKE_MAC_SDK.macosx.Path = $${MAC_SDK_PATH} QMAKE_MAC_SDK.macosx.platform_name = macosx QMAKE_MAC_SDK.macosx.SDKVersion = $${MAC_SDK_VERSION} QMAKE_MAC_SDK.macosx.PlatformPath = /phony QMAKE_APPLE_DEVICE_ARCHS=x86_64 !host_build: QMAKE_CFLAGS += -target $${MAC_TARGET} !host_build: QMAKE_OBJECTIVE_CFLAGS += $$QMAKE_CFLAGS !host_build: QMAKE_CXXFLAGS += $$QMAKE_CFLAGS -!host_build: QMAKE_LFLAGS += -target $${MAC_TARGET} -mlinker-version=$${MAC_LD64_VERSION} +!host_build: QMAKE_LFLAGS += -target $${MAC_TARGET} QMAKE_AR = $${CROSS_COMPILE}ar cq QMAKE_RANLIB=$${CROSS_COMPILE}ranlib QMAKE_LIBTOOL=$${CROSS_COMPILE}libtool QMAKE_INSTALL_NAME_TOOL=$${CROSS_COMPILE}install_name_tool load(qt_config) diff --git a/doc/build-osx.md b/doc/build-osx.md index 379db36986..508324f9ee 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -1,199 +1,99 @@ macOS Build Instructions and Notes ==================================== The commands in this guide should be executed in a Terminal application. The built-in one is located in `/Applications/Utilities/Terminal.app`. Preparation ----------- 1. Install Xcode from the app store if you don't have it already (it's a dependency for qt5) 2. Install the macOS command line tools: `xcode-select --install` When the popup appears, click `Install`. 3. Install [Homebrew](https://brew.sh). Dependencies ---------------------- Install dependencies: brew install berkeley-db boost cmake libevent miniupnpc ninja openssl protobuf python qrencode qt zeromq See [dependencies.md](dependencies.md) for a complete overview. If you want to build the disk image with `ninja osx-dmg` (.dmg / optional), you need RSVG: brew install librsvg Build Bitcoin ABC ----------------- Before you start building, please make sure that your compiler supports C++14. 1. Clone the Bitcoin ABC source code and cd into `bitcoin-abc` git clone https://github.com/Bitcoin-ABC/bitcoin-abc.git cd bitcoin-abc 2. Build Bitcoin ABC: Configure and build the headless Bitcoin ABC binaries as well as the GUI. You can disable the GUI build by passing `-DBUILD_BITCOIN_QT=OFF` to cmake. It is recommended to create a build directory to build out-of-tree. mkdir build cd build cmake -GNinja .. ninja 3. It is recommended to build and run the unit tests: ninja check 4. You can also create a .dmg that contains the .app bundle (optional): ninja osx-dmg Disable-wallet mode -------------------- When the intention is to run only a P2P node without a wallet, Bitcoin ABC may be compiled in disable-wallet mode with: cmake -GNinja .. -DBUILD_BITCOIN_WALLET=OFF Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call. Running ------- Bitcoin ABC is now available at `./src/bitcoind` Before running, it's recommended that you create an RPC configuration file: echo -e "rpcuser=bitcoinrpc\nrpcpassword=$(xxd -l 16 -p /dev/urandom)" > "/Users/${USER}/Library/Application Support/Bitcoin/bitcoin.conf" chmod 600 "/Users/${USER}/Library/Application Support/Bitcoin/bitcoin.conf" The first time you run bitcoind, it will start downloading the blockchain. This process could take many hours, or even days on slower than average systems. You can monitor the download process by looking at the debug.log file: tail -f $HOME/Library/Application\ Support/Bitcoin/debug.log Other commands: ------- ./src/bitcoind -daemon # Starts the bitcoin daemon. ./src/bitcoin-cli --help # Outputs a list of command-line options. ./src/bitcoin-cli help # Outputs a list of RPC commands when the daemon is running. Notes ----- * Building with downloaded Qt binaries is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714) - -Deterministic macOS DMG Notes ------------------------------ - -Working macOS DMGs are created in Linux by combining a recent clang, -the Apple binutils (ld, ar, etc) and DMG authoring tools. - -Apple uses clang extensively for development and has upstreamed the necessary -functionality so that a vanilla clang can take advantage. It supports the use -of -F, -target, -mmacosx-version-min, and --sysroot, which are all necessary -when building for macOS. - -Apple's version of binutils (called cctools) contains lots of functionality -missing in the FSF's binutils. In addition to extra linker options for -frameworks and sysroots, several other tools are needed as well such as -install_name_tool, lipo, and nmedit. These do not build under linux, so they -have been patched to do so. The work here was used as a starting point: -[mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). - -In order to build a working toolchain, the following source packages are needed -from Apple: cctools, dyld, and ld64. - -These tools inject timestamps by default, which produce non-deterministic -binaries. The ZERO_AR_DATE environment variable is used to disable that. - -This version of cctools has been patched to use the current version of clang's -headers and its libLTO.so rather than those from llvmgcc, as it was -originally done in toolchain4. - -To complicate things further, all builds must target an Apple SDK. These SDKs -are free to download, but not redistributable. -To obtain it, register for a developer account, then download the [Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). - -This file is several gigabytes in size, but only a single directory inside is -needed: -``` -Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk -``` - -Unfortunately, the usual linux tools (7zip, hpmount, loopback mount) are incapable of opening this file. -To create a tarball suitable for Gitian input, there are two options: - -Using macOS, you can mount the dmg, and then create it with: -``` - $ hdiutil attach Xcode_7.3.1.dmg - $ tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.11.sdk.tar.gz MacOSX10.11.sdk -``` - -Alternatively, you can use 7zip and SleuthKit to extract the files one by one. -The script contrib/macdeploy/extract-osx-sdk.sh automates this. First ensure -the dmg file is in the current directory, and then run the script. You may wish -to delete the intermediate 5.hfs file and MacOSX10.11.sdk (the directory) when -you've confirmed the extraction succeeded. - -```bash -apt-get install p7zip-full sleuthkit -contrib/macdeploy/extract-osx-sdk.sh -rm -rf 5.hfs MacOSX10.11.sdk -``` - -The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries -which are created using these tools. The build process has been designed to -avoid including the SDK's files in Gitian's outputs. All interim tarballs are -fully deterministic and may be freely redistributed. - -genisoimage is used to create the initial DMG. It is not deterministic as-is, -so it has been patched. A system genisoimage will work fine, but it will not -be deterministic because the file-order will change between invocations. -The patch can be seen here: [theuni/osx-cross-depends](https://raw.githubusercontent.com/theuni/osx-cross-depends/master/patches/cdrtools/genisoimage.diff). -No effort was made to fix this cleanly, so it likely leaks memory badly. But -it's only used for a single invocation, so that's no real concern. - -genisoimage cannot compress DMGs, so afterwards, the 'dmg' tool from the -libdmg-hfsplus project is used to compress it. There are several bugs in this -tool and its maintainer has seemingly abandoned the project. It has been forked -and is available (with fixes) here: [theuni/libdmg-hfsplus](https://github.com/theuni/libdmg-hfsplus). - -The 'dmg' tool has the ability to create DMGs from scratch as well, but this -functionality is broken. Only the compression feature is currently used. -Ideally, the creation could be fixed and genisoimage would no longer be necessary. - -Background images and other features can be added to DMG files by inserting a -.DS_Store before creation. This is generated by the script -contrib/macdeploy/custom_dsstore.py. - -As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a -requirement in order to satisfy the new Gatekeeper requirements. Because this -private key cannot be shared, we'll have to be a bit creative in order for the -build process to remain somewhat deterministic. Here's how it works: - -- Builders use Gitian to create an unsigned release. This outputs an unsigned - dmg which users may choose to bless and run. It also outputs an unsigned app - structure in the form of a tarball, which also contains all of the tools - that have been previously (deterministically) built in order to create a - final dmg. -- The Apple keyholder uses this unsigned app to create a detached signature, - using the script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). -- Builders feed the unsigned app + detached signature back into Gitian. It - uses the pre-built tools to recombine the pieces into a deterministic dmg. -