diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py new file mode 100755 --- /dev/null +++ b/contrib/gitian-build.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess +import sys + + +def setup(): + global args, workdir + programs = ['ruby', 'git', 'apt-cacher-ng', 'make', 'wget'] + if args.kvm: + programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils'] + elif args.docker: + dockers = ['docker.io', 'docker-ce'] + for i in dockers: + return_code = subprocess.call( + ['sudo', 'apt-get', 'install', '-qq', i]) + if return_code == 0: + break + if return_code != 0: + print('Cannot find any way to install docker', file=sys.stderr) + exit(1) + else: + programs += ['lxc', 'debootstrap'] + subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) + if not os.path.isdir('gitian-builder'): + subprocess.check_call( + ['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) + if not os.path.isdir('bitcoin-abc'): + subprocess.check_call( + ['git', 'clone', 'https://github.com/Bitcoin-ABC/bitcoin-abc.git']) + os.chdir('gitian-builder') + make_image_prog = ['bin/make-base-vm', + '--suite', 'bionic', '--arch', 'amd64'] + if args.docker: + make_image_prog += ['--docker'] + elif not args.kvm: + make_image_prog += ['--lxc'] + subprocess.check_call(make_image_prog) + os.chdir(workdir) + if args.is_bionic and not args.kvm and not args.docker: + subprocess.check_call( + ['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) + print('Reboot is required') + exit(0) + + +def build(): + global args, workdir + + os.makedirs('bitcoin-binaries/' + args.version, exist_ok=True) + print('\nBuilding Dependencies\n') + os.chdir('gitian-builder') + os.makedirs('inputs', exist_ok=True) + + subprocess.check_call(['make', '-C', '../bitcoin-abc/depends', + 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) + + if args.linux: + print('\nCompiling ' + args.version + ' Linux') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, + '--url', 'bitcoin='+args.url, '../bitcoin-abc/contrib/gitian-descriptors/gitian-linux.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version + + '-linux', '--destination', '../gitian.sigs/', '../bitcoin-abc/contrib/gitian-descriptors/gitian-linux.yml']) + subprocess.check_call( + 'mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/'+args.version, shell=True) + + if args.windows: + print('\nCompiling ' + args.version + ' Windows') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, + '--url', 'bitcoin='+args.url, '../bitcoin-abc/contrib/gitian-descriptors/gitian-win.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version + + '-win-unsigned', '--destination', '../gitian.sigs/', '../bitcoin-abc/contrib/gitian-descriptors/gitian-win.yml']) + subprocess.check_call( + 'mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/', shell=True) + subprocess.check_call( + 'mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe ../bitcoin-binaries/'+args.version, shell=True) + + if args.macos: + print('\nCompiling ' + args.version + ' MacOS') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, + '--url', 'bitcoin='+args.url, '../bitcoin-abc/contrib/gitian-descriptors/gitian-osx.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version + + '-osx-unsigned', '--destination', '../gitian.sigs/', '../bitcoin-abc/contrib/gitian-descriptors/gitian-osx.yml']) + subprocess.check_call( + 'mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/', shell=True) + subprocess.check_call( + 'mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ../bitcoin-binaries/'+args.version, shell=True) + + os.chdir(workdir) + + if args.commit_files: + print('\nCommitting '+args.version+' Unsigned Sigs\n') + os.chdir('gitian.sigs') + subprocess.check_call( + ['git', 'add', args.version+'-linux/'+args.signer]) + subprocess.check_call( + ['git', 'add', args.version+'-win-unsigned/'+args.signer]) + subprocess.check_call( + ['git', 'add', args.version+'-osx-unsigned/'+args.signer]) + subprocess.check_call( + ['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) + os.chdir(workdir) + + +def sign(): + global args, workdir + os.chdir('gitian-builder') + + if args.windows: + print('\nSigning ' + args.version + ' Windows') + subprocess.check_call('cp inputs/bitcoin-' + args.version + + '-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz', shell=True) + subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature='+args.commit, + '../bitcoin-abc/contrib/gitian-descriptors/gitian-win-signer.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-signed', + '--destination', '../gitian.sigs/', '../bitcoin-abc/contrib/gitian-descriptors/gitian-win-signer.yml']) + subprocess.check_call( + 'mv build/out/bitcoin-*win64-setup.exe ../bitcoin-binaries/'+args.version, shell=True) + subprocess.check_call( + 'mv build/out/bitcoin-*win32-setup.exe ../bitcoin-binaries/'+args.version, shell=True) + + if args.macos: + print('\nSigning ' + args.version + ' MacOS') + subprocess.check_call('cp inputs/bitcoin-' + args.version + + '-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz', shell=True) + subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature='+args.commit, + '../bitcoin-abc/contrib/gitian-descriptors/gitian-osx-signer.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-signed', + '--destination', '../gitian.sigs/', '../bitcoin-abc/contrib/gitian-descriptors/gitian-osx-signer.yml']) + subprocess.check_call('mv build/out/bitcoin-osx-signed.dmg ../bitcoin-binaries/' + + args.version+'/bitcoin-'+args.version+'-osx.dmg', shell=True) + + os.chdir(workdir) + + if args.commit_files: + print('\nCommitting '+args.version+' Signed Sigs\n') + os.chdir('gitian.sigs') + subprocess.check_call( + ['git', 'add', args.version+'-win-signed/'+args.signer]) + subprocess.check_call( + ['git', 'add', args.version+'-osx-signed/'+args.signer]) + subprocess.check_call(['git', 'commit', '-a', '-m', 'Add ' + + args.version+' signed binary sigs for '+args.signer]) + os.chdir(workdir) + + +def verify(): + global args, workdir + os.chdir('gitian-builder') + + print('\nVerifying v'+args.version+' Linux\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version + + '-linux', '../bitcoin-abc/contrib/gitian-descriptors/gitian-linux.yml']) + print('\nVerifying v'+args.version+' Windows\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version + + '-win-unsigned', '../bitcoin-abc/contrib/gitian-descriptors/gitian-win.yml']) + print('\nVerifying v'+args.version+' MacOS\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version + + '-osx-unsigned', '../bitcoin-abc/contrib/gitian-descriptors/gitian-osx.yml']) + print('\nVerifying v'+args.version+' Signed Windows\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version + + '-win-signed', '../bitcoin-abc/contrib/gitian-descriptors/gitian-win-signer.yml']) + print('\nVerifying v'+args.version+' Signed MacOS\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version + + '-osx-signed', '../bitcoin-abc/contrib/gitian-descriptors/gitian-osx-signer.yml']) + + os.chdir(workdir) + + +def main(): + global args, workdir + + parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version') + parser.add_argument('-c', '--commit', action='store_true', dest='commit', + help='Indicate that the version argument is for a commit or branch') + parser.add_argument('-p', '--pull', action='store_true', dest='pull', + help='Indicate that the version argument is the number of a github repository pull request') + parser.add_argument('-u', '--url', dest='url', default='https://github.com/Bitcoin-ABC/bitcoin-abc.git', + help='Specify the URL of the repository. Default is %(default)s') + parser.add_argument('-v', '--verify', action='store_true', + dest='verify', help='Verify the Gitian build') + parser.add_argument('-b', '--build', action='store_true', + dest='build', help='Do a Gitian build') + parser.add_argument('-s', '--sign', action='store_true', dest='sign', + help='Make signed binaries for Windows and MacOS') + parser.add_argument('-B', '--buildsign', action='store_true', + dest='buildsign', help='Build both signed and unsigned binaries') + parser.add_argument('-o', '--os', dest='os', default='lwm', + help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') + parser.add_argument('-j', '--jobs', dest='jobs', default='2', + help='Number of processes to use. Default %(default)s') + parser.add_argument('-m', '--memory', dest='memory', default='2000', + help='Memory to allocate in MiB. Default %(default)s') + parser.add_argument('-k', '--kvm', action='store_true', + dest='kvm', help='Use KVM instead of LXC') + parser.add_argument('-d', '--docker', action='store_true', + dest='docker', help='Use Docker instead of LXC') + parser.add_argument('-S', '--setup', action='store_true', dest='setup', + help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian)') + parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', + help='Create the assert file for detached signing. Will not commit anything.') + parser.add_argument('-n', '--no-commit', action='store_false', + dest='commit_files', help='Do not commit anything to git') + parser.add_argument( + 'signer', help='GPG signer to sign each build assert file') + parser.add_argument( + 'version', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') + + args = parser.parse_args() + workdir = os.getcwd() + + args.linux = 'l' in args.os + args.windows = 'w' in args.os + args.macos = 'm' in args.os + + args.is_bionic = b'bionic' in subprocess.check_output( + ['lsb_release', '-cs']) + + if args.buildsign: + args.build = True + args.sign = True + + if args.kvm and args.docker: + raise Exception('Error: cannot have both kvm and docker') + + args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' + + # Set environment variable USE_LXC or USE_DOCKER, let gitian-builder know that we use lxc or docker + if args.docker: + os.environ['USE_DOCKER'] = '1' + elif not args.kvm: + os.environ['USE_LXC'] = '1' + if not 'GITIAN_HOST_IP' in os.environ.keys(): + os.environ['GITIAN_HOST_IP'] = '10.0.3.1' + if not 'LXC_GUEST_IP' in os.environ.keys(): + os.environ['LXC_GUEST_IP'] = '10.0.3.5' + + # Disable for MacOS if no SDK found + if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): + print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') + args.macos = False + + script_name = os.path.basename(sys.argv[0]) + # Signer and version shouldn't be empty + if args.signer == '': + print(script_name+': Missing signer.') + print('Try '+script_name+' --help for more information') + exit(1) + if args.version == '': + print(script_name+': Missing version.') + print('Try '+script_name+' --help for more information') + exit(1) + + # Add leading 'v' for tags + if args.commit and args.pull: + raise Exception('Cannot have both commit and pull') + args.commit = ('' if args.commit else 'v') + args.version + + if args.setup: + setup() + + os.chdir('bitcoin') + if args.pull: + subprocess.check_call( + ['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) + os.chdir('../gitian-builder/inputs/bitcoin') + subprocess.check_call( + ['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) + args.commit = subprocess.check_output( + ['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True, encoding='utf8').strip() + args.version = 'pull-' + args.version + print(args.commit) + subprocess.check_call(['git', 'fetch']) + subprocess.check_call(['git', 'checkout', args.commit]) + os.chdir(workdir) + + if args.build: + build() + + if args.sign: + sign() + + if args.verify: + verify() + + +if __name__ == '__main__': + main() diff --git a/doc/gitian-building.md b/doc/gitian-building.md --- a/doc/gitian-building.md +++ b/doc/gitian-building.md @@ -1,7 +1,7 @@ Gitian building -================ +=============== -*Setup instructions for a Gitian build of Bitcoin ABC using an Ubuntu VM or physical system.* +*Setup instructions for a Gitian build of Bitcoin ABC using a VM or physical system.* Gitian is the deterministic build process that is used to build the Bitcoin ABC executables. It provides a way to be reasonably sure that the @@ -10,157 +10,103 @@ Multiple developers build the source code by following a specific descriptor ("recipe"), cryptographically sign the result, and upload the resulting signature. -These results are compared and only if they match, the build is accepted and uploaded -to bitcoinabc.org. +These results are compared and only if they match, the build is accepted and +uploaded to bitcoinabc.org. More independent Gitian builders are needed, which is why this guide exists. It is preferred you follow these steps yourself instead of using someone else's VM image to avoid 'contaminating' the build. Table of Contents ------------------- +----------------- - [Preparing the Gitian builder host](#preparing-the-gitian-builder-host) -- [Setting up the Gitian image](#setting-up-the-gitian-image) -- [Building Bitcoin ABC](#building-bitcoin-abc) +- [Getting and building the inputs](#getting-and-building-the-inputs) +- [Building Bitcoin Core](#building-bitcoin-core) +- [Building an alternative repository](#building-an-alternative-repository) +- [Signing externally](#signing-externally) +- [Uploading signatures](#uploading-signatures) Preparing the Gitian builder host --------------------------------- -The first step is to prepare the host environment that will be used to perform -the Gitian builds. This guide explains how to set up the environment, and how -to start the builds. - -The gitian build is easiest performed under Ubuntu Xenial. This guide will -focus on creating and using a vagrant box. However, you may run the provided -provision script on your favorite clean VM image using any virtualization -option, or a baremetal linux machine. If you wish to setup machine using -another technology, please see the provided provisioning script for gitian in -`contrib/gitian/provisioner.sh` - -Requirements: - - A machine with at least 64b of disk space - - 16GB of RAM - - Several installed tools: - - [Vagrant](https://www.vagrantup.com) - - [Packer](https://www.packer.io) - - [Virtualbox](https://www.virtualbox.org) - -After you have installed each of these tools, you will need to create an -ubuntu xenial vagrant "box." This is most easily done using the [box-cutter -project](https://github.com/boxcutter/ubuntu). (Note: Canonical provides a -vagrant box, however its disk space is insufficient for this guide.) +The first step is to prepare the host environment that will be used to perform the Gitian builds. +This guide explains how to set up the environment, and how to start the builds. -```bash -pushd -cd /tmp/ -git clone https://github.com/boxcutter/ubuntu.git -cd ubuntu -git checkout 7d1820c186d76122445c092bc2b872a8a94166ce -packer build -var-file=ubuntu1604.json -only=virtualbox-iso ubuntu.json -vagrant box add --name abc-xenial box/virtualbox/ubuntu1604-0.1.0.box -popd -``` +Gitian builds are known to be working on recent versions of Debian, Ubuntu and Fedora. +If your machine is already running one of those operating systems, you can perform Gitian builds on the actual hardware. +Alternatively, you can install one of the supported operating systems in a virtual machine. -After completion you should be able to run add the box to vagrant as "abc-xenial" -using `vagrant box add --name abc-xenial ` +You can create the virtual machine using [vagrant](./gitian-building/gitian-building-vagrant.md) or chose to setup the VM manually. -The final step for running vagrant is: +Any kind of virtualization can be used, for example: +- [VirtualBox](https://www.virtualbox.org/) (covered by this guide) +- [KVM](http://www.linux-kvm.org/page/Main_Page) +- [LXC](https://linuxcontainers.org/) -```bash -cd contrib/gitian/ -vagrant up -vagrant ssh -``` +Please refer to the following documents to set up the operating systems and Gitian. -This should drop you into a Xenial prompt as the user `vagrant`. +| | Debian | Fedora | +|-----------------------------------|------------------------------------------------------------------------------------|------------------------------------------------------------------------------------| +| Setup virtual machine (optional) | [Create Debian VirtualBox](./gitian-building/gitian-building-create-vm-debian.md) | [Create Fedora VirtualBox](./gitian-building/gitian-building-create-vm-fedora.md) | +| Setup Gitian | [Setup Gitian on Debian](./gitian-building/gitian-building-setup-gitian-debian.md) | [Setup Gitian on Fedora](./gitian-building/gitian-building-setup-gitian-fedora.md) | -Setting up the Gitian image ---------------------------- +Note that a version of `lxc-execute` higher or equal to 2.1.1 is required. +You can check the version with `lxc-execute --version`. -Gitian needs a virtual image of the operating system to build in. Currently -this is Ubuntu Xenial x86_64. This image will be copied and used every time -that a build is started to make sure that the build is deterministic. Creating -the image will take a while, but only has to be done once. +Non-Debian / Ubuntu, Manual and Offline Building +------------------------------------------------ +The instructions below use the automated script [gitian-build.py](https://github.com/Bitcoin-ABC/bitcoin-abc/blob/master/contrib/gitian-build.py) which only works in Debian/Ubuntu. For manual steps and instructions for fully offline signing, see [this guide](./gitian-building/gitian-building-manual.md). -Execute the following as user `vagrant`: +MacOS code signing +------------------ +In order to sign builds for MacOS, you need to download the free SDK and extract a file. The steps are described [here](./gitian-building/gitian-building-mac-os-sdk.md). + +It is possible to download the resulting archive directly for users that desire to do so: ```bash -cd gitian-builder -./bin/make-base-vm --lxc --distro debian --suite stretch --arch amd64 +curl -LO https://storage.googleapis.com/f4936e83b2dcbca742be51fb9692b153/MacOSX10.11.sdk.tar.gz +echo "4732b52b5ebe300c8c91cbeed6d19d59c1ff9c56c7a1dd6cfa518b9c2c72abde MacOSX10.11.sdk.tar.gz" | sha256sum -c ``` -There will be a lot of warnings printed during the build of the image. These -can be ignored. +Alternatively, you can skip the OSX build by adding `--os=lw` below. -Building Bitcoin ABC +Initial Gitian Setup -------------------- - -To build Bitcoin ABC (for Linux, OS X and Windows) run the following commands: +The `gitian-build.py` script will checkout different release tags, so it's best to copy it: ```bash -URL=https://github.com/bitcoin-abc/bitcoin-abc.git -COMMIT=v0.16.0 # or whatever release tag you wish - -# Note the path to descriptors assumes vagrant was used. These files are within the ABC repository normally. -./bin/gbuild --commit bitcoin=${COMMIT} --url bitcoin=${URL} /vagrant/contrib/gitian-descriptors/gitian-linux.yml -# Note: If you plan on signing the binaries generated during this process, be -# sure to copy them otherwise they will be overwritten by the next gbuild call. -cp -r ./build/out/* /vagrant/gitian/linux -# Also copy the manifest files in the same manner: -cp ./result/bitcoin-abc-*-linux-res.yml /vagrant/gitian/linux/ -./bin/gbuild --commit bitcoin=${COMMIT} --url bitcoin=${URL} /vagrant/contrib/gitian-descriptors/gitian-win.yml -cp -r ./build/out/* /vagrant/gitian/win -cp ./result/bitcoin-abc-*-win-res.yml /vagrant/gitian/win/ -./bin/gbuild --commit bitcoin=${COMMIT} --url bitcoin=${URL} /vagrant/contrib/gitian-descriptors/gitian-osx.yml -cp -r ./build/out/* /vagrant/gitian/osx -cp ./result/bitcoin-abc-*-osx-res.yml /vagrant/gitian/osx/ +cp bitcoin-abc/contrib/gitian-build.py . ``` -Note on the OSX build: If you encounter an error about a missing MacOSX10.11.sdk.tar.gz, then follow these steps: -``` -cd ./inputs -curl -LO https://storage.googleapis.com/f4936e83b2dcbca742be51fb9692b153/MacOSX10.11.sdk.tar.gz -``` +You only need to do this once: -Note: For executing gitian builds on local changes, change URL and COMMIT: ```bash -URL=/vagrant/ -COMMIT= # replace with your latest changes +./gitian-build.py --setup satoshi 0.18.5 ``` -This may take some time as it will build all the dependencies needed for each -descriptor. These dependencies will be cached after a successful build to -avoid rebuilding them when possible. +Where `satoshi` is your Github name and `0.18.5` is the most recent tag (without `v`). -At any time you can check the package installation and build progress with +Build binaries +-------------- +Windows and OSX have code signed binaries, but those won't be available until a few developers have gitian signed the non-codesigned binaries. + +To build the most recent tag: ```bash -tail -f var/install.log -tail -f var/build.log +./gitian-build.py --detach-sign --no-commit -b satoshi 0.18.5 ``` -Output from `gbuild` will look something like - - Initialized empty Git repository in /home/vagrant/gitian-builder/inputs/bitcoin/.git/ - remote: Counting objects: 57959, done. - remote: Total 57959 (delta 0), reused 0 (delta 0), pack-reused 57958 - Receiving objects: 100% (57959/57959), 53.76 MiB | 484.00 KiB/s, done. - Resolving deltas: 100% (41590/41590), done. - From https://github.com/bitcoin/bitcoin - ... (new tags, new branch etc) - --- Building for trusty amd64 --- - Stopping target if it is up - Making a new image copy - stdin: is not a tty - Starting target - Checking if target is up - Preparing build environment - Updating apt-get repository (log in var/install.log) - Installing additional packages (log in var/install.log) - Grabbing package manifest - stdin: is not a tty - Creating build script (var/build-script) - lxc-start: Connection refused - inotify event with no name (mask 32768) - Running build script (log in var/build.log) +To speed up the build, use `-j 5 -m 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. + +If all went well, this produces a number of (uncommited) `.assert` files in the gitian.sigs repository. + +You need to copy these uncommited changes to your host machine, where you can sign them: + +```bash +export NAME=satoshi +gpg --output $VERSION-linux/$NAME/bitcoin-abc-linux-0.18.5-build.assert.sig --detach-sign 0.18.5-linux/$NAME/bitcoin-abc-linux-0.18.5-build.assert +gpg --output $VERSION-osx-unsigned/$NAME/bitcoin-abc-osx-0.18.5-build.assert.sig --detach-sign 0.18.5-osx-unsigned/$NAME/bitcoin-abc-osx-0.18.5-build.assert +gpg --output $VERSION-win-unsigned/$NAME/bitcoin-abc-win-0.18.5-build.assert.sig --detach-sign 0.18.5-win-unsigned/$NAME/bitcoin-abc-win-0.18.5-build.assert +``` diff --git a/doc/gitian-building/figs/all_files_in_one_partition.png b/doc/gitian-building/figs/all_files_in_one_partition.png new file mode 100644 index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 GIT binary patch literal 0 Hc$@ *All files in one partition* + +![](figs/all_files_in_one_partition.png) + + - Finish partitioning and write changes to disk -> *Yes* (`Tab`, `Enter` to select the `Yes` button) + +![](figs/debian_install_14_finish.png) +![](figs/debian_install_15_write_changes.png) + +- The base system will be installed, this will take a minute or so +- Choose a mirror (any will do) + +![](figs/debian_install_16_choose_a_mirror.png) + +- Enter proxy information (unless you are on an intranet, leave this empty) + +![](figs/debian_install_18_proxy_settings.png) + +- Wait a bit while 'Select and install software' runs +- Participate in popularity contest -> *No* +- Choose software to install. We need just the base system. +- Make sure only 'SSH server' and 'Standard System Utilities' are checked +- Uncheck 'Debian Desktop Environment' and 'Print Server' + +![](figs/debian_install_19_software_selection.png) + +- Install the GRUB boot loader to the master boot record? -> Yes + +![](figs/debian_install_20_install_grub.png) + +- Device for boot loader installation -> ata-VBOX_HARDDISK + +![](figs/debian_install_21_install_grub_bootloader.png) + +- Installation Complete -> *Continue* +- After installation, the VM will reboot and you will have a working Debian VM. Congratulations! + +![](figs/debian_install_22_finish_installation.png) + +Connecting to the VM +-------------------- + +After the VM has booted you can connect to it using SSH, and files can be copied from and to the VM using a SFTP utility. +Connect to `localhost`, port `22222` (or the port configured when installing the VM). +On Windows you can use [putty](http://www.chiark.greenend.org.uk/~sgtatham/putty/download.html) and [WinSCP](http://winscp.net/eng/index.php). + +For example, to connect as `gitianuser` from a Linux command prompt use + + $ ssh gitianuser@localhost -p 22222 + The authenticity of host '[localhost]:22222 ([127.0.0.1]:22222)' can't be established. + RSA key fingerprint is ae:f5:c8:9f:17:c6:c7:1b:c2:1b:12:31:1d:bb:d0:c7. + Are you sure you want to continue connecting (yes/no)? yes + Warning: Permanently added '[localhost]:22222' (RSA) to the list of known hosts. + gitianuser@localhost's password: (enter gitianuser password configured during install) + + The programs included with the Debian GNU/Linux system are free software; + the exact distribution terms for each program are described in the + individual files in /usr/share/doc/*/copyright. + + Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent + permitted by applicable law. + gitianuser@debian:~$ + +Use `sudo` to execute commands as root. + +Optional - Easier login to the VM +--------------------------------- + +You'll need to generate an SSH key, e.g. by following the instructions under "Generating a new SSH key" [here](https://help.github.com/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent). + +After that, login to the VM and enter: + +```bash +mkdir .ssh +``` + +On your machine edit or create `~/.ssh/config` and add: + +```bash +Host gitian + HostName localhost + Port 22222 + User gitianuser +``` + +Open a new terminal tab and enter: + +```bash +scp ~/.ssh/id_rsa.pub gitian:.ssh/authorized_keys +``` + +Next time you need to login to the VM, just use: `ssh gitian` diff --git a/doc/gitian-building/gitian-building-create-vm-fedora.md b/doc/gitian-building/gitian-building-create-vm-fedora.md new file mode 100644 --- /dev/null +++ b/doc/gitian-building/gitian-building-create-vm-fedora.md @@ -0,0 +1,133 @@ +# Setup Fedora virtual machine on VirtualBox + +Table of Contents +------------------ + +- [Create a new VirtualBox VM](#create-a-new-virtualbox-vm) +- [Connecting to the VM](#connecting-to-the-vm) + +Create a new VirtualBox VM +--------------------------- +In the VirtualBox GUI click "New" and choose the following parameters in the wizard: + +![](figs/create_new_vm_fedora.png) + +- Type: Linux, Fedora (64-bit) + +![](figs/create_vm_memsize.png) + +- Memory Size: at least 3000MB, anything less and the build might not complete. + +![](figs/create_vm_hard_disk.png) + +- Hard Disk: Create a virtual hard disk now + +![](figs/create_vm_hard_disk_file_type.png) + +- Hard Disk file type: Use the default, VDI (VirtualBox Disk Image) + +![](figs/create_vm_storage_physical_hard_disk.png) + +- Storage on physical hard disk: Dynamically Allocated + +![](figs/create_vm_file_location_size.png) + +- File location and size: at least 40GB +- Click `Create` + +After creating the VM, we need to configure it. + +- Click the `Settings` button, then go to `System` tab and `Processor` sub-tab. Increase the number of processors to the number of cores on your machine if you want builds to be faster. + +![](figs/system_settings.png) + +- Go to the `Network` tab. Adapter 1 should be attached to `NAT`. + +![](figs/network_settings.png) + +- Click `Advanced`, then `Port Forwarding`. We want to set up a port through which we can reach the VM to get files in and out. +- Create a new rule by clicking the plus icon. + +![](figs/port_forwarding_rules.png) + +- Set up the new rule the following way: + - Name: `SSH` + - Protocol: `TCP` + - Leave Host IP empty + - Host Port: `22222` + - Leave Guest IP empty + - Guest Port: `22` + +- Click `Ok` twice to save. + +Get the [Fedora Netinstall Image](https://download.fedoraproject.org/pub/fedora/linux/releases/29/Workstation/x86_64/iso/Fedora-Workstation-netinst-x86_64-29-1.2.iso) (other resonably recent release should work as well, see [Fedora Workstation Download page](https://getfedora.org/en/workstation/download/)). +This DVD image can be [validated](https://getfedora.org/verify) using a SHA256 hashing tool, for example on +Unixy OSes by entering the following in a terminal: + + echo "6554404b66d38b89693232966d9290ed62156e32d1edde4074b1d25c97a7b10e Fedora-Workstation-netinst-x86_64-29-1.2.iso" | sha256sum -c + # (must return OK) + +Replace `sha256sum` with `shasum` on OSX. + +Then start the VM. On the first launch you will be asked for a CD or DVD image. Choose the downloaded ISO. + +![](figs/select_startup_disk_fedora.png) + +Installing Fedora +------------------ + +This section will explain how to install Fedora on the newly created VM. + +- Choose the default option. This will start the graphical installer. + +![](figs/fedora_install_1_boot_menu.png) + +- Pick the language, locale and keyboard settings of your choice. You can just go with the defaults or select your own information. + +- The VM will detect network settings using DHCP, this should all proceed automatically +- Afterward, the installer will automatically pick an appropriate *Installation Source* (closest mirror) and starts fetching the package metadata. +- Then, click the button for *Software Selection* and choose the *Minimal Install* from as the base environment and no addons for the selected environment. This will reduce installation time and disk usage. +- Finally, open the settings for *Installation Destination* and click *Done*. The default settings are just fine. + +- The *Installation Summary* should look like this: + +![](figs/fedora_install_2_summary.png) + +- Confirm by clicking on *Begin Installation*. +- You can leave the root password empty. Otherwise, enter it twice and remember it for later. +- Also, create a new user. Name the new user `gitianuser` (the full name doesn't matter, you can leave it empty) +- Select *Make this user administrator* to add them to the wheel group. +- Choose a user password and enter it twice (remember it for later) + +![](figs/fedora_install_3_users.png) + +![](figs/fedora_install_4_users.png) + +- Complete the installation by clicking *Finish configuration*. Then wait for it to complete and select *Reboot*. +- If the greeter asks you to install fedora, make sure to remove the ISO: In the running VirtualBox instance, select *Devices* > *Optical Drives* > *Remove disk from virtual drive*. Then reboot again. +- After reboot you will have a working Fedora VM. Congratulations! + +You'll be presented with a screen similar to this. + +![](figs/fedora_root_login.png) + +Connecting to the VM +---------------------- + +After the VM has booted you can connect to it using SSH, and files can be copied from and to the VM using a SFTP utility. +Connect to `localhost`, port `22222` (or the port configured when installing the VM). +On Windows you can use [putty](http://www.chiark.greenend.org.uk/~sgtatham/putty/download.html) and [WinSCP](http://winscp.net/eng/index.php). + +For example, to connect as `gitianuser` from a Linux command prompt use + +``` +$ ssh gitianuser@localhost -p 22222 +The authenticity of host '[localhost]:22222 ([127.0.0.1]:22222)' can't be established. +ECDSA key fingerprint is SHA256:Px5l4jnEItv44YnaESVlPMTu9gPYopIELKok99CjCvo. +Are you sure you want to continue connecting (yes/no)? yes +Warning: Permanently added '[localhost]:22222' (ECDSA) to the list of known hosts. +gitianuser@localhost's password: (enter gitianuser password configured during install) +[gitianuser@localhost ~]$ +``` + +Use `sudo` to execute commands as root. diff --git a/doc/gitian-building/gitian-building-mac-os-sdk.md b/doc/gitian-building/gitian-building-mac-os-sdk.md new file mode 100644 --- /dev/null +++ b/doc/gitian-building/gitian-building-mac-os-sdk.md @@ -0,0 +1,56 @@ +Gitian building Mac OS SDK +========================== + +On the host machine, register for a free Apple [developer account](https://developer.apple.com/register/), then download the SDK [here](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). + +MacOS host +---------- + +Using Mac OS X, you can mount the dmg, and then extract the SDK with: +``` + $ hdiutil attach Xcode_7.3.1.dmg + $ tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.11.sdk.tar.gz MacOSX10.11.sdk +``` + +Clean up the files you don't need: + +```sh +diskutil unmount /Volumes/Xcode +rm Xcode_7.3.1.dmg +``` + +Non-MacOS host: +--------------- + +Alternatively, you can use 7zip and SleuthKit to extract the files one by one. +The script [extract-osx-sdk.sh](https://github.com/Bitcoin-ABC/bitcoin-abc/blob/master/contrib/macdeploy/extract-osx-sdk.sh) automates this. First ensure +the dmg file is in the current directory, and then run the script. + +You may wish to delete the intermediate 5.hfs file and MacOSX10.11.sdk (the directory) when +you've confirmed the extraction succeeded. + +```bash +apt-get install p7zip-full sleuthkit +contrib/macdeploy/extract-osx-sdk.sh +rm -rf 5.hfs MacOSX10.11.sdk +``` + +Copy SDK to Gitian VM: +---------------------- +Copy it to the Gitian VM and clean up, e.g.: + +```bash +scp MacOSX10.11.sdk.tar.gz gitian: +rm MacOSX10.11.sdk.tar.gz +``` + +Login to the VM and: + +```bash +mkdir -p gitian-builder/inputs +mv MacOSX10.11.sdk.tar.gz gitian-builder/inputs +``` + +Troubleshooting +--------------- +See [README_osx.md](https://github.com/Bitcoin-ABC/bitcoin-abc/blob/master/doc/README_osx.md) for troubleshooting tips. diff --git a/doc/gitian-building/gitian-building-manual.md b/doc/gitian-building/gitian-building-manual.md new file mode 100644 --- /dev/null +++ b/doc/gitian-building/gitian-building-manual.md @@ -0,0 +1,116 @@ +Getting and building the inputs +------------------------------- + +At this point you have two options, you can either use the automated script (found in [https://github.com/Bitcoin-ABC/bitcoin-abc/blob/master/contrib/gitian-build.py](https://github.com/Bitcoin-ABC/bitcoin-abc/blob/master/contrib/gitian-build.py), only works in Debian/Ubuntu) or you could manually do everything by following this guide. +If you are using the automated script, then run it with the `--setup` command. Afterwards, run it with the `--build` command (example: `contrib/gitian-build.py -b signer 0.15.0`). Otherwise ignore this. + +Follow the instructions in [https://github.com/bitcoin/bitcoin/blob/master/doc/release-process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/release-process.md#fetch-and-create-inputs-first-time-or-when-dependency-versions-change) +in the bitcoin repository under 'Fetch and create inputs' to install sources which require +manual intervention. Also optionally follow the next step: 'Seed the Gitian sources cache +and offline git repositories' which will fetch the remaining files required for building +offline. + +Building Bitcoin ABC +-------------------- + +To build Bitcoin ABC (for Linux, OS X and Windows) just follow the steps under 'perform +Gitian builds' in [https://github.com/bitcoin/bitcoin/blob/master/doc/release-process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/release-process.md#setup-and-perform-gitian-builds) in the bitcoin repository. + +This may take some time as it will build all the dependencies needed for each descriptor. +These dependencies will be cached after a successful build to avoid rebuilding them when possible. + +At any time you can check the package installation and build progress with + +```bash +tail -f var/install.log +tail -f var/build.log +``` + +Output from `gbuild` will look something like + + Initialized empty Git repository in /home/gitianuser/gitian-builder/inputs/bitcoin/.git/ + remote: Counting objects: 57959, done. + remote: Total 57959 (delta 0), reused 0 (delta 0), pack-reused 57958 + Receiving objects: 100% (57959/57959), 53.76 MiB | 484.00 KiB/s, done. + Resolving deltas: 100% (41590/41590), done. + From https://github.com/Bitcoin-ABC/bitcoin-abc.git + ... (new tags, new branch etc) + --- Building for trusty amd64 --- + Stopping target if it is up + Making a new image copy + stdin: is not a tty + Starting target + Checking if target is up + Preparing build environment + Updating apt-get repository (log in var/install.log) + Installing additional packages (log in var/install.log) + Grabbing package manifest + stdin: is not a tty + Creating build script (var/build-script) + lxc-start: Connection refused - inotify event with no name (mask 32768) + Running build script (log in var/build.log) + +Building an alternative repository +---------------------------------- + +If you want to do a test build of a pull on GitHub it can be useful to point +the Gitian builder at an alternative repository, using the same descriptors +and inputs. + +For example: +```bash +URL=https://github.com/Bitcoin-ABC/bitcoin-abc.git +COMMIT=v0.18.5 +./bin/gbuild --commit bitcoin=${COMMIT} --url bitcoin=${URL} ../bitcoin-abc/contrib/gitian-descriptors/gitian-linux.yml +./bin/gbuild --commit bitcoin=${COMMIT} --url bitcoin=${URL} ../bitcoin-abc/contrib/gitian-descriptors/gitian-win.yml +./bin/gbuild --commit bitcoin=${COMMIT} --url bitcoin=${URL} ../bitcoin-abc/contrib/gitian-descriptors/gitian-osx.yml +``` + +Building fully offline +---------------------- + +For building fully offline including attaching signatures to unsigned builds, the detached-sigs repository +and the bitcoin git repository with the desired tag must both be available locally, and then gbuild must be +told where to find them. It also requires an apt-cacher-ng which is fully-populated but set to offline mode, or +manually disabling gitian-builder's use of apt-get to update the VM build environment. + +To configure apt-cacher-ng as an offline cacher, you will need to first populate its cache with the relevant +files. You must additionally patch target-bin/bootstrap-fixup to set its apt sources to something other than +plain archive.ubuntu.com: us.archive.ubuntu.com works. + +So, if you use LXC: + +```bash +export PATH="$PATH":${HOME}/gitian-builder/libexec +export USE_LXC=1 +cd ~/gitian-builder +./libexec/make-clean-vm --suite stretch --arch amd64 + +LXC_ARCH=amd64 LXC_SUITE=stretch on-target -u root dpkg --add-architecture i386 +LXC_ARCH=amd64 LXC_SUITE=stretch on-target -u root apt-get update +LXC_ARCH=amd64 LXC_SUITE=stretch on-target -u root \ + -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install \ + $( sed -ne '/^packages:/,/^[^-]/ {/^- .*/{s/"//g;s/- //;p}}' ../bitcoin-abc/contrib/gitian-descriptors/*|sort|uniq ) +LXC_ARCH=amd64 LXC_SUITE=stretch on-target -u root apt-get -q -y purge grub +LXC_ARCH=amd64 LXC_SUITE=stretch on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade +``` + +And then set offline mode for apt-cacher-ng: + +``` +/etc/apt-cacher-ng/acng.conf +[...] +Offlinemode: 1 +[...] + +sudo service apt-cacher-ng restart +``` + +Then when building, override the remote URLs that gbuild would otherwise pull from the Gitian descriptors:: +```bash +cd ~ +BTCPATH=${HOME}/bitcoin-abc +COMMIT= + +./bin/gbuild --commit bitcoin=${COMMIT} --url bitcoin=${BTCPATH} ${BTCPATH}/contrib/gitian-descriptors/gitian-win-signer.yml +``` diff --git a/doc/gitian-building/gitian-building-setup-gitian-debian.md b/doc/gitian-building/gitian-building-setup-gitian-debian.md new file mode 100644 --- /dev/null +++ b/doc/gitian-building/gitian-building-setup-gitian-debian.md @@ -0,0 +1,110 @@ +Table of Contents +----------------- + +- [Setting up Debian for Gitian building](#setting-up-debian-for-gitian-building) +- [Installing Gitian](#installing-gitian) +- [Setting up the Gitian image](#setting-up-the-gitian-image) + + +Setting up Debian for Gitian building +------------------------------------- + +In this section we will be setting up the Debian installation for Gitian building. +We assume that a user `gitianuser` was previously added. + +First we need to set up dependencies. Type/paste the following in the terminal: + +```bash +sudo apt-get install git ruby apt-cacher-ng qemu-utils debootstrap lxc python-cheetah parted kpartx bridge-utils make ubuntu-archive-keyring curl firewalld apparmor +``` + +Then set up LXC and the rest with the following, which is a complex jumble of settings and workarounds: + +```bash +sudo -s +# the version of lxc-start in Debian needs to run as root, so make sure +# that the build script can execute it without providing a password +echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-start" > /etc/sudoers.d/gitian-lxc +echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-execute" >> /etc/sudoers.d/gitian-lxc +# make /etc/rc.local script that sets up bridge between guest and host +echo '#!/bin/sh -e' > /etc/rc.local +echo 'brctl addbr br0' >> /etc/rc.local +echo 'ip addr add 10.0.3.1/24 broadcast 10.0.3.255 dev br0' >> /etc/rc.local +echo 'ip link set br0 up' >> /etc/rc.local +echo 'firewall-cmd --zone=trusted --add-interface=br0' >> /etc/rc.local +echo 'exit 0' >> /etc/rc.local +chmod +x /etc/rc.local +# make sure that USE_LXC is always set when logging in as gitianuser, +# and configure LXC IP addresses +echo 'export USE_LXC=1' >> /home/gitianuser/.profile +echo 'export GITIAN_HOST_IP=10.0.3.1' >> /home/gitianuser/.profile +echo 'export LXC_GUEST_IP=10.0.3.5' >> /home/gitianuser/.profile +reboot +``` + +At the end Debian is rebooted to make sure that the changes take effect. The steps in this +section only need to be performed once. + +**Note**: When sudo asks for a password, enter the password for the user `gitianuser` not for `root`. + +Installing Gitian +----------------- + +Re-login as the user `gitianuser` that was created during installation. +The rest of the steps in this guide will be performed as that user. + +There is no `python-vm-builder` package in Debian, so we need to install it from source ourselves, + +```bash +wget http://archive.ubuntu.com/ubuntu/pool/universe/v/vm-builder/vm-builder_0.12.4+bzr494.orig.tar.gz +echo "76cbf8c52c391160b2641e7120dbade5afded713afaa6032f733a261f13e6a8e vm-builder_0.12.4+bzr494.orig.tar.gz" | sha256sum -c +# (verification -- must return OK) +tar -zxvf vm-builder_0.12.4+bzr494.orig.tar.gz +cd vm-builder-0.12.4+bzr494 +sudo python setup.py install +cd .. +``` + +**Note**: When sudo asks for a password, enter the password for the user `gitianuser` not for `root`. + +Clone the git repositories for bitcoin and Gitian. + +```bash +git clone https://github.com/devrandom/gitian-builder.git +git clone https://github.com/Bitcoin-ABC/bitcoin-abc.git +``` + +Setting up the Gitian image +--------------------------- + +Gitian needs a virtual image of the operating system to build in. +Currently this is Ubuntu Bionic x86_64, however previous releases were built +with Ubuntu Trusty x86_64. +This image will be copied and used every time that a build is started to +make sure that the build is deterministic. +Creating the image will take a while, but only has to be done once. + +Execute the following as user `gitianuser`: + +```bash +cd gitian-builder +bin/make-base-vm --lxc --arch amd64 --distro debian --suite stretch +``` + +There will be a lot of warnings printed during the build of the image. These can be ignored. + +**Note**: When sudo asks for a password, enter the password for the user `gitianuser` not for `root`. + +Downloading dependencies +------------------------ + +Gitian may have problems connecting to the internet, which can make the build +flacky. In order to avoid this, it is possible to download dependencies ahead +of time. + +Execute the following as user `gitianuser`: + +```bash +cd gitian-builder +make -C ../bitcoin-abc/depends download SOURCES_PATH=`pwd`/cache/common +``` diff --git a/doc/gitian-building/gitian-building-setup-gitian-fedora.md b/doc/gitian-building/gitian-building-setup-gitian-fedora.md new file mode 100644 --- /dev/null +++ b/doc/gitian-building/gitian-building-setup-gitian-fedora.md @@ -0,0 +1,73 @@ +Table of Contents +------------------ + +- [Setting up Fedora for Gitian building](#setting-up-fedora-for-gitian-building) +- [Installing Gitian](#installing-gitian) +- [Setting up the Gitian image](#setting-up-the-gitian-image) + + +Setting up Fedora for Gitian building +-------------------------------------- + +In this section we will be setting up the Fedora installation for Gitian building. +We assume that a user `gitianuser` was previously created and added to the `wheel` group. + +First we need to set up dependencies. Type/paste the following in the terminal: + +```bash +sudo dnf install git ruby gnupg docker tar rsync wget curl +``` + +Then set up docker and the rest with the following, which is a complex jumble of settings and workarounds: + +```bash +sudo -s +systemctl enable docker.service +# the version of docker in Fedora needs to run as root, so make sure +# that the build script can execute it without providing a password +# https://docs.docker.com/install/linux/linux-postinstall/ +groupadd docker +usermod -aG docker gitianuser +echo 'export USE_DOCKER=1' >> /home/gitianuser/.bash_profile +reboot +``` + +At the end Fedora is rebooted to make sure that the changes take effect. The steps in this +section only need to be performed once. + +**Note**: When sudo asks for a password, enter the password for the user `gitianuser` not for `root`. + +Installing Gitian +------------------ + +Login as the user `gitianuser` that was created during installation. +The rest of the steps in this guide will be performed as that user. + +Clone the git repositories for bitcoin and Gitian. + +```bash +git clone https://github.com/devrandom/gitian-builder.git +git clone https://github.com/bitcoin/bitcoin +git clone https://github.com/bitcoin-core/gitian.sigs.git +git clone https://github.com/bitcoin-core/bitcoin-detached-sigs.git +``` + +Setting up the Gitian image +------------------------- + +Gitian needs a virtual image of the operating system to build in. +Currently this is Ubuntu Bionic x86_64, however previous releases were built +with Ubuntu Trusty x86_64. +This image will be copied and used every time that a build is started to +make sure that the build is deterministic. +Creating the image will take a while, but only has to be done once. + +Execute the following as user `gitianuser`: + +```bash +cd gitian-builder +bin/make-base-vm --docker --arch amd64 --suite bionic # For releases after and including 0.17.0 +bin/make-base-vm --docker --arch amd64 --suite trusty # For releases before 0.17.0 +``` + +**Note**: When sudo asks for a password, enter the password for the user `gitianuser` not for `root`. diff --git a/doc/gitian-building/gitian-building-vargant.md b/doc/gitian-building/gitian-building-vargant.md new file mode 100644 --- /dev/null +++ b/doc/gitian-building/gitian-building-vargant.md @@ -0,0 +1,68 @@ +Table of Contents +----------------- + +- [Preparing the Gitian builder host](#preparing-the-gitian-builder-host) +- [Setting up the Gitian image](#setting-up-the-gitian-image) + + +Preparing the Gitian builder host +--------------------------------- + +The first step is to prepare the host environment that will be used to perform +the Gitian builds. This guide explains how to set up the environment, and how +to start the builds. + +Requirements: + - A machine with at least 64b of disk space + - 16GB of RAM + - Several installed tools: + - [Vagrant](https://www.vagrantup.com) + - [Packer](https://www.packer.io) + - [Virtualbox](https://www.virtualbox.org) + +After you have installed each of these tools, you will need to create an +ubuntu xenial vagrant "box." This is most easily done using the [box-cutter +project](https://github.com/boxcutter/ubuntu). (Note: Canonical provides a +vagrant box, however its disk space is insufficient for this guide.) + +```bash +pushd +cd /tmp/ +git clone https://github.com/boxcutter/ubuntu.git +cd ubuntu +git checkout 7d1820c186d76122445c092bc2b872a8a94166ce +packer build -var-file=ubuntu1604.json -only=virtualbox-iso ubuntu.json +vagrant box add --name abc-xenial box/virtualbox/ubuntu1604-0.1.0.box +popd +``` + +After completion you should be able to run add the box to vagrant as "abc-xenial" +using `vagrant box add --name abc-xenial ` + +The final step for running vagrant is: + +```bash +cd contrib/gitian/ +vagrant up +vagrant ssh +``` + +This should drop you into a Xenial prompt as the user `vagrant`. + +Setting up the Gitian image +--------------------------- + +Gitian needs a virtual image of the operating system to build in. Currently +this is Debian Stretch x86_64. This image will be copied and used every time +that a build is started to make sure that the build is deterministic. Creating +the image will take a while, but only has to be done once. + +Execute the following as user `vagrant`: + +```bash +cd gitian-builder +./bin/make-base-vm --lxc --distro debian --suite stretch --arch amd64 +``` + +There will be a lot of warnings printed during the build of the image. These +can be ignored.