diff --git a/contrib/teamcity/gitian.sh b/contrib/teamcity/gitian.sh --- a/contrib/teamcity/gitian.sh +++ b/contrib/teamcity/gitian.sh @@ -1,24 +1,13 @@ #!/bin/bash -e -wget http://archive.ubuntu.com/ubuntu/pool/universe/v/vm-builder/vm-builder_0.12.4+bzr494.orig.tar.gz -echo "76cbf8c52c391160b2641e7120dbade5afded713afaa6032f733a261f13e6a8e vm-builder_0.12.4+bzr494.orig.tar.gz" | sha256sum -c -# (verification -- must return OK) -tar -zxvf vm-builder_0.12.4+bzr494.orig.tar.gz -cd vm-builder-0.12.4+bzr494 -sudo python setup.py install -cd .. - -## Install Gitian -git clone https://github.com/devrandom/gitian-builder.git - -export COMMIT=HEAD +export COMMIT=`git rev-parse HEAD` export URL=`pwd` export USE_LXC=1 export LXC_BRIDGE=lxcbr0 export GITIAN_HOST_IP=10.0.3.1 export LXC_GUEST_IP=10.0.3.5 -cd gitian-builder +cd ~/gitian-builder if [[ "${OS_NAME}" == "osx" ]]; then wget https://storage.googleapis.com/f4936e83b2dcbca742be51fb9692b153/MacOSX10.11.sdk.tar.gz @@ -31,8 +20,7 @@ ## Determine the number of build threads THREADS=$(nproc || sysctl -n hw.ncpu) -./bin/make-base-vm --lxc --distro debian --suite stretch --arch amd64 -./bin/gbuild -j${THREADS} -m3500 --commit bitcoin=${COMMIT} --url bitcoin=${URL} ../contrib/gitian-descriptors/gitian-${OS_NAME}.yml +./bin/gbuild -j${THREADS} -m3500 --commit bitcoin=${COMMIT} --url bitcoin=${URL} ${URL}/contrib/gitian-descriptors/gitian-${OS_NAME}.yml cd .. mkdir ${OS_NAME} diff --git a/contrib/teamcity/setup-agent.sh b/contrib/teamcity/setup-agent.sh new file mode 100755 --- /dev/null +++ b/contrib/teamcity/setup-agent.sh @@ -0,0 +1,333 @@ +#!/usr/bin/env bash + +set -e + +PORT="22" +PUBKEY="${HOME}/.ssh/id_rsa.pub" +MIRROR="http://deb.debian.org" + +function printHelp() { + echo "Usage: ${0} [-h|--help] [-k|--pubkey pubkey] [-m|--mirror mirror] [-p|--port sshport] remote" + echo " -h|--help: print this help" + echo " -k|--pubkey: path to the public key to send to the server for SSH authentication (default: ${PUBKEY})" + echo " -m|--mirror: Debian mirror to get the packages from (default: ${MIRROR})" + echo " -p|--port: SSH connection port number (default: ${PORT})" + echo " remote: hostname or IP of the remote to setup" + echo + echo "You need root access to the remote you want to setup." + echo "This script may require you to enter your password." + exit ${1} +} + +[ $# -lt 1 ] || [ $# -gt 7 ] && echo "Invalid parameters" && printHelp 1 + +# Parse command line arguments +while [[ $# -gt 0 ]]; do +case "$1" in + -h|--help) + printHelp 0 + ;; + -k|--pubkey) + PUBKEY="$2" + shift # shift past argument + shift # shift past value + ;; + -m|--mirror) + MIRROR="$2" + shift # shift past argument + shift # shift past value + ;; + -p|--port) + PORT="$2" + shift # shift past argument + shift # shift past value + ;; + *) + REMOTE=${1} + shift # shift past value + ;; +esac +done + +[ ! -f "${PUBKEY}" ] && echo "Pubkey file not found" && printHelp 1 + +[ -z "${REMOTE}" ] && echo "Missing remote" && printHelp 1 + +# sshpass is mandatory +which sshpass > /dev/null || (echo "ERROR: 'sshpass' program not found, please install it." && exit 1) + +echo "Login as root to ${REMOTE}..." +echo -n "root@${REMOTE}'s password: " +read -s PASSWORD +echo + +sshpass -p "${PASSWORD}" ssh -oStrictHostKeyChecking=no -p "${PORT}" "root@${REMOTE}" "MIRROR=${MIRROR}" "/usr/bin/env bash" <<"ENDOFSSH" + + +### CHECK THE SCRIPT IS RUNNING ON DEBIAN 9, WARN OTHERWISE +# Debian based distributions put definitions in /etc/os-release +DIST_FILE=/etc/os-release + +DIST_WARNING="WARNING: this script is targeting Debian 9 only. Use at your own risk." + +# Additional packages to install +if [[ ! -f "${DIST_FILE}" ]]; then + echo "WARNING: unable to locate the distribution definitions file ${DIST_FILE}." + echo "${DIST_WARNING}" +fi + +DIST_ID=`cat ${DIST_FILE} | grep "^ID=" | cut -d'=' -f2` +DIST_VERSION_ID=`cat ${DIST_FILE} | grep "^VERSION_ID=" | cut -d'=' -f2` + +echo "Distribution: ${DIST_ID}" +echo "Version: ${DIST_VERSION_ID}" + +if [[ "${DIST_ID}" != "debian" || "${DIST_VERSION_ID}" != '"9"' ]]; then + echo "${DIST_WARNING}" +fi + + +### FIND THE WAN NETWORK INTERFACE +echo "INFO: start WAN network interface detection" + +NET_PING_SITE="google.com" + +# Get the active network interfaces +NET_DEVS=`ip addr show | awk '/inet.*brd/{print $NF}'` +for d in ${NET_DEVS} +do + echo "INFO: trying network interface $d" + ping -c 1 -I $d "${NET_PING_SITE}" + if [[ $? -eq 0 ]]; then + NET_DEV=$d + break + fi +done + +if [[ -z "${NET_DEV}" ]]; then + echo "ERROR: could not determine an active WAN network interface. Aborting" + exit 1 +fi + +echo "INFO: using ${NET_DEV} as the main network interface" + + +### INSTALL ADDITIONAL PACKAGES +echo "INFO: start installing additional packages" + +set -e + +INSTALL_STRETCH_PACKAGES=( + # Server management + bash-completion + fail2ban + git + less + sudo + vim + + # Build dependencies + automake + autotools-dev + bsdmainutils + build-essential + ccache + default-jdk-headless + libboost-all-dev + libdb-dev + libdb++-dev + libminiupnpc-dev + libprotobuf-dev + libqrencode-dev + libqt5core5a + libqt5dbus5 + libqt5gui5 + libssl-dev + libtool + libzmq3-dev + pkg-config + protobuf-compiler + python3 + python3-zmq + qttools5-dev + qttools5-dev-tools + + # Teamcity dependencies + default-jre + unzip + + # Gitian dependencies + apparmor + apt-cacher-ng + bridge-utils + curl + debootstrap + firewalld + iptables + kpartx + lxc + make + parted + python-cheetah + qemu-utils + ruby + ubuntu-archive-keyring +) + +# Some package might be too old on stable, pick them from testing +INSTALL_BUSTER_PACKAGES=( + # libevent 2.0.21 from stable has a memory leak issue, install 2.1.8 instead + # https://bugzilla.mozilla.org/show_bug.cgi?id=1021350 + libevent-dev +) + +echo "Package: *" >> /etc/apt/preferences.d/stable.pref +echo "Pin: release a=stable" >> /etc/apt/preferences.d/stable.pref +echo "Pin-Priority: 900" >> /etc/apt/preferences.d/stable.pref + +echo "Package: *" >> /etc/apt/preferences.d/testing.pref +echo "Pin: release a=testing" >> /etc/apt/preferences.d/testing.pref +echo "Pin-Priority: 400" >> /etc/apt/preferences.d/testing.pref + +echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/99defaultrelease + +echo "deb ${MIRROR}/debian/ stretch main" >> /etc/apt/sources.list.d/stable.list +echo "deb-src ${MIRROR}/debian/ stretch main" >> /etc/apt/sources.list.d/stable.list +echo "deb ${MIRROR}/debian-security/ stretch/updates main" >> /etc/apt/sources.list.d/stable.list +echo "deb-src ${MIRROR}/debian-security/ stretch/updates main" >> /etc/apt/sources.list.d/stable.list +echo "deb ${MIRROR}/debian/ stretch-updates main" >> /etc/apt/sources.list.d/stable.list +echo "deb-src ${MIRROR}/debian/ stretch-updates main" >> /etc/apt/sources.list.d/stable.list + +echo "deb ${MIRROR}/debian/ buster main" >> /etc/apt/sources.list.d/testing.list +echo "deb-src ${MIRROR}/debian/ buster main" >> /etc/apt/sources.list.d/testing.list + +function join_by { local IFS="$1"; shift; echo "$*"; } + +echo "INFO: updating package list" +apt-get update + +echo "INFO: installing from stable: ${INSTALL_STRETCH_PACKAGES[*]}" +apt-get install -y $(join_by ' ' ${INSTALL_STRETCH_PACKAGES[@]}) + +echo "INFO: installing from testing: ${INSTALL_BUSTER_PACKAGES[*]}" +apt-get install -y -t buster $(join_by ' ' ${INSTALL_BUSTER_PACKAGES[@]}) + + +### TEAMCITY SETUP +echo "INFO: start Teamcity agent setup" + +# Add the Teamcity user (no password) +useradd -m teamcity +adduser teamcity sudo +passwd -d teamcity + +# Setup Java environment variable +echo 'JAVA_HOME="/usr/lib/jvm/default-java"' >> /etc/environment + +# Move to the teamcity user home directory +cd /home/teamcity + +# Get the Teamcity agent script and configure the agent +sudo -u teamcity wget https://build.bitcoinabc.org/update/buildAgent.zip +sudo -u teamcity echo "4e0e5409ffd81b2a3605f824e2d59887efab2659717a4558fbb5ea386a2a5ed9 buildAgent.zip" | sha256sum -c +sudo -u teamcity unzip -d buildAgent -q buildAgent.zip +rm buildAgent.zip +cd buildAgent/conf +sudo -u teamcity cp buildAgent.dist.properties buildAgent.properties +sudo -u teamcity sed -i "s#serverUrl=.*#serverUrl=https://build.bitcoinabc.org/#g" buildAgent.properties +sudo -u teamcity sed -i "s#name=.*#name=`hostname`#g" buildAgent.properties +cd ../bin +sudo -u teamcity chmod +x agent.sh + +# Setup automatic start for the Teamcity agent +cd /etc/init.d +wget https://raw.githubusercontent.com/Bitcoin-ABC/bitcoin-abc/master/contrib/teamcity/buildAgent-autostart -O buildAgent +echo "03b35e5af5d943c7b5228ac1ac945e0114829d563424d356e3f58d67aaa93e33 buildAgent" | sha256sum -c +chmod 755 buildAgent +update-rc.d buildAgent defaults + + +### GITIAN SETUP + +# the version of lxc-start in Debian needs to run as root, so make sure +# that the build script can execute it without providing a password +echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-start" > /etc/sudoers.d/gitian-lxc +echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-execute" >> /etc/sudoers.d/gitian-lxc + +# make /etc/rc.local script that sets up bridge between guest and host +echo '#!/bin/sh -e' > /etc/rc.local +echo 'brctl addbr lxcbr0' >> /etc/rc.local +echo 'ip addr add 10.0.3.1/24 broadcast 10.0.3.255 dev lxcbr0' >> /etc/rc.local +echo 'ip link set lxcbr0 up' >> /etc/rc.local +echo 'firewall-cmd --zone=trusted --add-interface=lxcbr0' >> /etc/rc.local +echo "iptables -t nat -A POSTROUTING -o ${NET_DEV} -j MASQUERADE" >> /etc/rc.local +echo 'echo 1 > /proc/sys/net/ipv4/ip_forward' >> /etc/rc.local +echo 'exit 0' >> /etc/rc.local +chmod +x /etc/rc.local + +# make sure that USE_LXC is always set when logging in as teamcity, +# and configure LXC IP addresses +echo 'export USE_LXC=1' >> /home/teamcity/.profile +echo 'export GITIAN_HOST_IP=10.0.3.1' >> /home/teamcity/.profile +echo 'export LXC_GUEST_IP=10.0.3.5' >> /home/teamcity/.profile + +# Install vm-builder +cd /home/teamcity +sudo -u teamcity wget http://archive.ubuntu.com/ubuntu/pool/universe/v/vm-builder/vm-builder_0.12.4+bzr494.orig.tar.gz +sudo -u teamcity echo "76cbf8c52c391160b2641e7120dbade5afded713afaa6032f733a261f13e6a8e vm-builder_0.12.4+bzr494.orig.tar.gz" | sha256sum -c +sudo -u teamcity tar -zxvf vm-builder_0.12.4+bzr494.orig.tar.gz +rm vm-builder_0.12.4+bzr494.orig.tar.gz +cd vm-builder-0.12.4+bzr494 +python setup.py install +cd .. + +# Prepare Gitian base VM +sudo -u teamcity git clone https://github.com/devrandom/gitian-builder.git +sudo -u teamcity git clone https://github.com/Bitcoin-ABC/bitcoin-abc.git +cd gitian-builder +sudo -u teamcity bin/make-base-vm --lxc --arch amd64 --distro debian --suite stretch + +# Set teamcity password to "teamcity". This will only used once to send the public key to the server +usermod -p '$6$UInKh8jI$x.W6.XTe40pqFWW7JcUgPmjpIqAoDZ53b4NNvPd0/mGKRr/Nd23KVpF3N2kFwp354Vb6axMWK5zf1s6VJ8kLA1' teamcity + +ENDOFSSH + +# Copy user public key to server +sshpass -p "teamcity" ssh-copy-id -i "${PUBKEY}" -p "${PORT}" "teamcity@${REMOTE}" + +ssh -q -p "${PORT}" "teamcity@${REMOTE}" exit || ( + echo "ERROR: Connection to ${REMOTE} as teamcity failed !" + echo "Please check the key used to authenticate." + echo "A new 'teamcity' user has been added with password 'teamcity'." + echo "You may want to connect to ${REMOTE} and delete this user." + exit 2 +) + +sshpass -p "${PASSWORD}" ssh -p "${PORT}" "root@${REMOTE}" "/usr/bin/env bash" <<"ENDOFSSH" + + +### SETTING LOGIN PERMISSIONS +echo "INFO: Disabling login via password and disabling root login" +echo "INFO: Only login as the teamcity user with keys is allowed" + +# Reset teamcity password to something more secure +usermod -p '$6$pHcBHB0i$wPFaojwrPdlYl9mWnAiiWoSFkwDvJq6mTMrP5AP.JSaVZVm7RF..P7wx5a3hQsJf9tcH.1M8OHc7IMndlDNlM.' teamcity + +SSHD_CONFIG_FILE=/etc/ssh/sshd_config + +# Disable password login +grep -q "^PasswordAuthentication" "${SSHD_CONFIG_FILE}" \ + && sed -ri "s/^PasswordAuthentication.*/PasswordAuthentication no/g" "${SSHD_CONFIG_FILE}" \ + || echo "PasswordAuthentication no" >> "${SSHD_CONFIG_FILE}" + +# Disable root login +grep -q "^PermitRootLogin" "${SSHD_CONFIG_FILE}" \ + && sed -ri "s/^PermitRootLogin.*/PermitRootLogin no/g" "${SSHD_CONFIG_FILE}" \ + || echo "PermitRootLogin no" >> "${SSHD_CONFIG_FILE}" + + +### REBOOT +echo "INFO: rebooting the machine" +reboot + +ENDOFSSH