diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 54b24c691..c6b2649aa 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -1,178 +1,190 @@ #!/usr/bin/env python3 # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' A script to check that the (Linux) executables produced by gitian only contain allowed gcc, glibc and libstdc++ version symbols. This makes sure they are still compatible with the minimum supported Linux distribution versions. Example usage: find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py ''' import subprocess import re import sys import os # Debian 8.11 (Jessie) has: # # - g++ version 4.9.2 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=g%2B%2B) # - libc version 2.19.18 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libc6) # - libstdc++ version 4.8.4 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libstdc%2B%2B6) # # Ubuntu 14.04 (Trusty Tahr) has: # # - g++ version 4.8.2 (https://packages.ubuntu.com/search?suite=trusty§ion=all&arch=any&keywords=g%2B%2B&searchon=names) # - libc version 2.19.0 (https://packages.ubuntu.com/search?suite=trusty§ion=all&arch=any&keywords=libc6&searchon=names) # - libstdc++ version 4.8.2 (https://packages.ubuntu.com/search?suite=trusty§ion=all&arch=any&keywords=libstdc%2B%2B&searchon=names) # # Taking the minimum of these as our target. # # According to GNU ABI document (http://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: # GCC 4.8.0: GCC_4.8.0 # GCC 4.8.0: GLIBCXX_3.4.18, CXXABI_1.3.7 # (glibc) GLIBC_2_19 # MAX_VERSIONS = { - 'GCC': (4, 8, 0), - 'CXXABI': (1, 3, 7), - 'GLIBCXX': (3, 4, 18), - 'GLIBC': (2, 19) + 'GCC': (4, 8, 0), + 'CXXABI': (1, 3, 7), + 'GLIBCXX': (3, 4, 18), + 'GLIBC': (2, 19), + 'LIBATOMIC': (1, 0) } # See here for a description of _IO_stdin_used: # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { - '_edata', '_end', '_init', '__bss_start', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', + '_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', # Figure out why we get these symbols exported on xenial. '_ZNKSt5ctypeIcE8do_widenEc', 'in6addr_any', 'optarg', '_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_destroyEv' } READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') # Allowed NEEDED libraries ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt 'libgcc_s.so.1', # GCC base support 'libc.so.6', # C library 'libpthread.so.0', # threading 'libanl.so.1', # DNS resolve 'libm.so.6', # math library 'librt.so.1', # real-time (clock) + 'libatomic.so.1', 'ld-linux-x86-64.so.2', # 64-bit dynamic linker 'ld-linux.so.2', # 32-bit dynamic linker + 'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker + 'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker # bitcoin-qt only 'libX11-xcb.so.1', # part of X11 'libX11.so.6', # part of X11 'libxcb.so.1', # part of X11 'libfontconfig.so.1', # font support 'libfreetype.so.6', # font parsing 'libdl.so.2' # programming interface to dynamic linker } +ARCH_MIN_GLIBC_VER = { + '80386': (2, 1), + 'X86-64': (2, 2, 5), + 'ARM': (2, 4), + 'AArch64': (2, 17) +} class CPPFilt(object): ''' Demangle C++ symbol names. Use a pipe to the 'c++filt' command. ''' def __init__(self): self.proc = subprocess.Popen( CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) def __call__(self, mangled): self.proc.stdin.write(mangled + '\n') self.proc.stdin.flush() return self.proc.stdout.readline().rstrip() def close(self): self.proc.stdin.close() self.proc.stdout.close() self.proc.wait() def read_symbols(executable, imports=True): ''' Parse an ELF executable and return a list of (symbol,version) tuples for dynamic, imported symbols. ''' - p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, + p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', '-h', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Could not read symbols for {}: {}'.format( executable, stderr.strip())) syms = [] for line in stdout.splitlines(): line = line.split() + if 'Machine:' in line: + arch = line[-1] if len(line) > 7 and re.match('[0-9]+:$', line[0]): (sym, _, version) = line[7].partition('@') is_import = line[6] == 'UND' if version.startswith('@'): version = version[1:] if is_import == imports: - syms.append((sym, version)) + syms.append((sym, version, arch)) return syms -def check_version(max_versions, version): +def check_version(max_versions, version, arch): if '_' in version: (lib, _, ver) = version.rpartition('_') else: lib = version ver = '0' ver = tuple([int(x) for x in ver.split('.')]) if not lib in max_versions: return False - return ver <= max_versions[lib] + return ver <= max_versions[lib] or lib == 'GLIBC' and ver <= ARCH_MIN_GLIBC_VER[arch] def read_libraries(filename): p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') libraries = [] for line in stdout.splitlines(): tokens = line.split() if len(tokens) > 2 and tokens[1] == '(NEEDED)': match = re.match( r'^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) if match: libraries.append(match.group(1)) else: raise ValueError('Unparseable (NEEDED) specification') return libraries if __name__ == '__main__': cppfilt = CPPFilt() retval = 0 for filename in sys.argv[1:]: # Check imported symbols - for sym, version in read_symbols(filename, True): - if version and not check_version(MAX_VERSIONS, version): + for sym, version, arch in read_symbols(filename, True): + if version and not check_version(MAX_VERSIONS, version, arch): print('{}: symbol {} from unsupported version {}'.format( filename, cppfilt(sym), version)) retval = 1 # Check exported symbols - for sym, version in read_symbols(filename, False): + for sym, version, arch in read_symbols(filename, False): if sym in IGNORE_EXPORTS: continue print('{}: export of symbol {} not allowed'.format( filename, cppfilt(sym))) retval = 1 # Check dependency libraries for library_name in read_libraries(filename): if library_name not in ALLOWED_LIBRARIES: print('{}: NEEDED library {} is not allowed'.format( filename, library_name)) retval = 1 sys.exit(retval) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 704089aed..48589a63d 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,188 +1,178 @@ --- name: "bitcoin-abc-0.20-linux" enable_cache: true distro: "debian" suites: - "buster" architectures: - "amd64" multiarch: - "i386" packages: - "curl" - "gperf" - "g++-aarch64-linux-gnu" - "gcc-aarch64-linux-gnu" - "binutils-aarch64-linux-gnu" - "g++-arm-linux-gnueabihf" - "gcc-arm-linux-gnueabihf" - "binutils-arm-linux-gnueabihf" - "binutils-gold" - "build-essential" - "git" - "pkg-config" - "autoconf" - "libtool" - "automake" - "faketime" - "bsdmainutils" - "ca-certificates" - "python" - "lib32stdc++-8-dev" - "libc6-dev:i386" remotes: - "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git" "dir": "bitcoin" files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu" CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="date ar ranlib nm" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" HOST_LDFLAGS=-static-libstdc++ export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export GZIP="-9n" export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" create_per-host_faketime_wrappers "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} EXTRA_INCLUDES_BASE=$WRAP_DIR/extra_includes mkdir -p $EXTRA_INCLUDES_BASE # x86 needs /usr/include/i386-linux-gnu/asm pointed to /usr/include/x86_64-linux-gnu/asm, # but we can't write there. Instead, create a link here and force it to be included in the # search paths by wrapping gcc/g++. mkdir -p $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu rm -f $WRAP_DIR/extra_includes/i686-pc-linux-gnu/asm ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu/asm for prog in gcc g++; do rm -f ${WRAP_DIR}/${prog} cat << EOF > ${WRAP_DIR}/${prog} #!/usr/bin/env bash REAL="`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1`" for var in "\$@" do if [ "\$var" = "-m32" ]; then export C_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" break fi done \$REAL \$@ EOF chmod +x ${WRAP_DIR}/${prog} done cd bitcoin BASEPREFIX=`pwd`/depends # Build dependencies for each host for i in $HOSTS; do EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" if [ -d "$EXTRA_INCLUDES" ]; then export HOST_ID_SALT="$EXTRA_INCLUDES" fi make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" unset HOST_ID_SALT done # Faketime for binaries export PATH=${PATH_orig} create_global_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} # Create the release tarball using (arbitrarily) the first host ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist SOURCEDIST=`echo bitcoin-abc-*.tar.gz` DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` # Correct tar file order mkdir -p temp pushd temp tar xf ../$SOURCEDIST find bitcoin-abc-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST popd ORIGPATH="$PATH" # Extract the release tarball into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH=`pwd`/installed/${DISTNAME} mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security - - #TODO: This is a quick hack that disables symbol checking for arm. - # Instead, we should investigate why these are popping up. - # For aarch64, we'll need to bump up the min GLIBC version, as the abi - # support wasn't introduced until 2.17. - case $i in - aarch64-*) : ;; - arm-*) : ;; - *) make ${MAKEOPTS} -C src check-symbols ;; - esac - + make ${MAKEOPTS} -C src check-symbols make install DESTDIR=${INSTALLPATH} cd installed find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig find ${DISTNAME}/bin -type f -executable -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; find ${DISTNAME}/lib -type f -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz cd ../../ rm -rf distsrc-${i} done mkdir -p $OUTDIR/src mv $SOURCEDIST $OUTDIR/src