Page MenuHomePhabricator

No OneTemporary

diff --git a/cmake/modules/BinaryTest.cmake b/cmake/modules/BinaryTest.cmake
index 73cb9492b..9553bab6a 100644
--- a/cmake/modules/BinaryTest.cmake
+++ b/cmake/modules/BinaryTest.cmake
@@ -1,41 +1,40 @@
# Facilities to run tests on the executable binaries.
macro(add_to_symbols_check BINARY)
if(NOT TARGET symbol-check)
add_custom_target(symbol-check)
endif()
set(CUSTOM_TARGET_NAME "symbol-check-${BINARY}")
add_custom_target("${CUSTOM_TARGET_NAME}"
COMMENT "Running symbol-check.py on ${BINARY}..."
COMMAND
"OTOOL=${CMAKE_OTOOL}"
"${Python_EXECUTABLE}"
"${CMAKE_SOURCE_DIR}/contrib/devtools/symbol-check.py"
"$<TARGET_FILE:${BINARY}>"
DEPENDS
"${BINARY}"
)
add_dependencies(symbol-check "${CUSTOM_TARGET_NAME}")
endmacro()
macro(add_to_security_check BINARY)
if(NOT TARGET security-check)
add_custom_target(security-check)
endif()
set(CUSTOM_TARGET_NAME "security-check-${BINARY}")
add_custom_target("${CUSTOM_TARGET_NAME}"
COMMENT "Running security-check.py on ${BINARY}..."
COMMAND
- "OTOOL=${CMAKE_OTOOL}"
"${Python_EXECUTABLE}"
"${CMAKE_SOURCE_DIR}/contrib/devtools/security-check.py"
"$<TARGET_FILE:${BINARY}>"
DEPENDS
"${BINARY}"
)
add_dependencies(security-check "${CUSTOM_TARGET_NAME}")
endmacro()
diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py
index 4b4c06f32..8dc5b44b7 100755
--- a/contrib/devtools/security-check.py
+++ b/contrib/devtools/security-check.py
@@ -1,312 +1,255 @@
#!/usr/bin/env python3
# Copyright (c) 2015-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
'''
Perform basic security checks on a series of executables.
Exit status will be 0 if successful, and the program will be silent.
Otherwise the exit status will be 1 and it will log which executables failed which checks.
-Needs `objdump` (for PE) and `otool` (for MACHO).
'''
-import subprocess
import sys
-import os
from typing import List, Optional
+import lief
import pixie
-OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump')
-OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool')
-
-
-def run_command(command) -> str:
- p = subprocess.run(
- command,
- stdout=subprocess.PIPE,
- check=True,
- universal_newlines=True)
- return p.stdout
-
def check_ELF_PIE(executable) -> bool:
'''
Check for position independent executable (PIE), allowing for address space
randomization.
'''
elf = pixie.load(executable)
return elf.hdr.e_type == pixie.ET_DYN
def check_ELF_NX(executable) -> bool:
'''
Check that no sections are writable and executable (including the stack)
'''
elf = pixie.load(executable)
have_wx = False
have_gnu_stack = False
for ph in elf.program_headers:
if ph.p_type == pixie.PT_GNU_STACK:
have_gnu_stack = True
# section is both writable and executable
if (ph.p_flags & pixie.PF_W) != 0 and (ph.p_flags & pixie.PF_X) != 0:
have_wx = True
return have_gnu_stack and not have_wx
def check_ELF_RELRO(executable) -> bool:
'''
Check for read-only relocations.
GNU_RELRO program header must exist
Dynamic section must have BIND_NOW flag
'''
elf = pixie.load(executable)
have_gnu_relro = False
for ph in elf.program_headers:
# Note: not checking p_flags == PF_R: here as linkers set the permission
# differently. This does not affect security: the permission flags of the
# GNU_RELRO program header are ignored, the PT_LOAD header determines the
# effective permissions.
# However, the dynamic linker need to write to this area so these are RW.
# Glibc itself takes care of mprotecting this area R after relocations are
# finished. See also https://marc.info/?l=binutils&m=1498883354122353
if ph.p_type == pixie.PT_GNU_RELRO:
have_gnu_relro = True
have_bindnow = False
for flags in elf.query_dyn_tags(pixie.DT_FLAGS):
assert isinstance(flags, int)
if flags & pixie.DF_BIND_NOW:
have_bindnow = True
return have_gnu_relro and have_bindnow
def check_ELF_Canary(executable) -> bool:
'''
Check for use of stack canary
'''
elf = pixie.load(executable)
for symbol in elf.dyn_symbols:
if symbol.name == b'__stack_chk_fail':
return True
return False
def check_ELF_separate_code(executable):
'''
Check that sections are appropriately separated in virtual memory,
based on their permissions. This checks for missing -Wl,-z,separate-code
and potentially other problems.
'''
elf = pixie.load(executable)
R = pixie.PF_R
W = pixie.PF_W
E = pixie.PF_X
EXPECTED_FLAGS = {
# Read + execute
b'.init': R | E,
b'.plt': R | E,
b'.plt.got': R | E,
b'.plt.sec': R | E,
b'.text': R | E,
b'.fini': R | E,
# Read-only data
b'.interp': R,
b'.note.gnu.property': R,
b'.note.gnu.build-id': R,
b'.note.ABI-tag': R,
b'.gnu.hash': R,
b'.dynsym': R,
b'.dynstr': R,
b'.gnu.version': R,
b'.gnu.version_r': R,
b'.rela.dyn': R,
b'.rela.plt': R,
b'.rodata': R,
b'.eh_frame_hdr': R,
b'.eh_frame': R,
b'.qtmetadata': R,
b'.gcc_except_table': R,
b'.stapsdt.base': R,
# Writable data
b'.init_array': R | W,
b'.fini_array': R | W,
b'.dynamic': R | W,
b'.got': R | W,
b'.data': R | W,
b'.bss': R | W,
}
if elf.hdr.e_machine == pixie.EM_PPC64:
# .plt is RW on ppc64 even with separate-code
EXPECTED_FLAGS[b'.plt'] = R | W
# For all LOAD program headers get mapping to the list of sections,
# and for each section, remember the flags of the associated program header.
flags_per_section = {}
for ph in elf.program_headers:
if ph.p_type == pixie.PT_LOAD:
for section in ph.sections:
assert(section.name not in flags_per_section)
flags_per_section[section.name] = ph.p_flags
# Spot-check ELF LOAD program header flags per section
# If these sections exist, check them against the expected R/W/E flags
for (section, flags) in flags_per_section.items():
if section in EXPECTED_FLAGS:
if EXPECTED_FLAGS[section] != flags:
return False
return True
-def get_PE_dll_characteristics(executable) -> int:
- '''Get PE DllCharacteristics bits'''
- stdout = run_command([OBJDUMP_CMD, '-x', executable])
-
- bits = 0
- for line in stdout.splitlines():
- tokens = line.split()
- if len(tokens) >= 2 and tokens[0] == 'DllCharacteristics':
- bits = int(tokens[1], 16)
- return bits
-
-
-IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020
-IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE = 0x0040
-IMAGE_DLL_CHARACTERISTICS_NX_COMPAT = 0x0100
-
-
def check_PE_DYNAMIC_BASE(executable) -> bool:
'''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)'''
- bits = get_PE_dll_characteristics(executable)
- return (bits & IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE) == IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE
+ binary = lief.parse(executable)
+ return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists
# Must support high-entropy 64-bit address space layout randomization
# in addition to DYNAMIC_BASE to have secure ASLR.
def check_PE_HIGH_ENTROPY_VA(executable) -> bool:
'''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR'''
- bits = get_PE_dll_characteristics(executable)
- return (bits & IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA) == IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA
+ binary = lief.parse(executable)
+ return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists
def check_PE_RELOC_SECTION(executable) -> bool:
'''Check for a reloc section. This is required for functional ASLR.'''
- stdout = run_command([OBJDUMP_CMD, '-h', executable])
-
- for line in stdout.splitlines():
- if '.reloc' in line:
- return True
- return False
-
-
-def check_PE_NX(executable) -> bool:
- '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)'''
- bits = get_PE_dll_characteristics(executable)
- return (
- bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT
-
+ binary = lief.parse(executable)
+ return binary.has_relocations
-def get_MACHO_executable_flags(executable) -> List[str]:
- stdout = run_command([OTOOL_CMD, '-vh', executable])
- flags: List[str] = []
- for line in stdout.splitlines():
- tokens = line.split()
- # filter first two header lines
- if 'magic' in tokens or 'Mach' in tokens:
- continue
- # filter ncmds and sizeofcmds values
- flags += [t for t in tokens if not t.isdigit()]
- return flags
-
-
-def check_MACHO_PIE(executable) -> bool:
+def check_MACHO_NOUNDEFS(executable) -> bool:
'''
- Check for position independent executable (PIE), allowing for address space randomization.
+ Check for no undefined references.
'''
- flags = get_MACHO_executable_flags(executable)
- return 'PIE' in flags
+ binary = lief.parse(executable)
+ return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS)
-def check_MACHO_NOUNDEFS(executable) -> bool:
+def check_MACHO_Canary(executable) -> bool:
'''
- Check for no undefined references.
+ Check for use of stack canary
'''
- flags = get_MACHO_executable_flags(executable)
- return 'NOUNDEFS' in flags
+ binary = lief.parse(executable)
+ return binary.has_symbol('___stack_chk_fail')
-def check_MACHO_NX(executable) -> bool:
+def check_PIE(executable) -> bool:
'''
- Check for no stack execution
+ Check for position independent executable (PIE),
+ allowing for address space randomization.
'''
- flags = get_MACHO_executable_flags(executable)
- return 'ALLOW_STACK_EXECUTION' not in flags
+ binary = lief.parse(executable)
+ return binary.is_pie
-def check_MACHO_Canary(executable) -> bool:
+def check_NX(executable) -> bool:
'''
- Check for use of stack canary
+ Check for no stack execution
'''
- stdout = run_command([OTOOL_CMD, '-Iv', executable])
- for line in stdout.splitlines():
- if '___stack_chk_fail' in line:
- return True
- return False
+ binary = lief.parse(executable)
+ return binary.has_nx
CHECKS = {
'ELF': [
('PIE', check_ELF_PIE),
('NX', check_ELF_NX),
('RELRO', check_ELF_RELRO),
('Canary', check_ELF_Canary),
('separate_code', check_ELF_separate_code),
],
'PE': [
+ ('PIE', check_PIE),
('DYNAMIC_BASE', check_PE_DYNAMIC_BASE),
('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA),
- ('NX', check_PE_NX),
+ ('NX', check_NX),
('RELOC_SECTION', check_PE_RELOC_SECTION),
],
'MACHO': [
- ('PIE', check_MACHO_PIE),
+ ('PIE', check_PIE),
('NOUNDEFS', check_MACHO_NOUNDEFS),
- ('NX', check_MACHO_NX),
+ ('NX', check_NX),
('Canary', check_MACHO_Canary),
]
}
def identify_executable(executable) -> Optional[str]:
with open(filename, 'rb') as f:
magic = f.read(4)
if magic.startswith(b'MZ'):
return 'PE'
elif magic.startswith(b'\x7fELF'):
return 'ELF'
elif magic.startswith(b'\xcf\xfa'):
return 'MACHO'
return None
if __name__ == '__main__':
- retval = 0
+ retval: int = 0
for filename in sys.argv[1:]:
try:
etype = identify_executable(filename)
if etype is None:
- print('{}: unknown format'.format(filename))
+ print(f'{filename}: unknown format')
retval = 1
continue
- failed = []
+ failed: List[str] = []
for (name, func) in CHECKS[etype]:
if not func(filename):
failed.append(name)
if failed:
- print('{}: failed {}'.format(filename, ' '.join(failed)))
+ print(f"{filename}: failed {' '.join(failed)}")
retval = 1
except IOError:
- print('{}: cannot open'.format(filename))
+ print(f'{filename}: cannot open')
retval = 1
sys.exit(retval)
diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml
index 393992a04..f9dd3485a 100644
--- a/contrib/gitian-descriptors/gitian-linux.yml
+++ b/contrib/gitian-descriptors/gitian-linux.yml
@@ -1,201 +1,204 @@
---
name: "bitcoin-abc-linux"
enable_cache: true
distro: "debian"
suites:
- "bullseye"
architectures:
- "amd64"
packages:
- "autoconf"
- "automake"
- "binutils-aarch64-linux-gnu"
- "binutils-arm-linux-gnueabihf"
- "binutils-gold"
- "bsdmainutils"
- "ca-certificates"
- "cmake"
- "curl"
- "faketime"
# Use gcc/g++ 9 to avoid introducing the new pthread_cond_clockwait from glibc
# 2.30, which would make our release binary incompatible with systems using an
# older glibc version.
- "g++-9"
- "g++-9-aarch64-linux-gnu"
- "g++-9-arm-linux-gnueabihf"
- "gcc-9"
- "gcc-9-aarch64-linux-gnu"
- "gcc-9-arm-linux-gnueabihf"
- "git"
- "gperf"
- "libtool"
- "ninja-build"
- "pkg-config"
- "python3"
+- "python3-pip"
remotes:
- "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git"
"dir": "bitcoin"
files: []
script: |
WRAP_DIR=$HOME/wrapped
HOSTS=(
x86_64-linux-gnu
arm-linux-gnueabihf
aarch64-linux-gnu
)
# CMake toolchain file name differ from host name
declare -A CMAKE_TOOLCHAIN_FILE
CMAKE_TOOLCHAIN_FILE[x86_64-linux-gnu]=Linux64.cmake
CMAKE_TOOLCHAIN_FILE[arm-linux-gnueabihf]=LinuxARM.cmake
CMAKE_TOOLCHAIN_FILE[aarch64-linux-gnu]=LinuxAArch64.cmake
# Allow extra cmake option to be specified for each host
declare -A CMAKE_EXTRA_OPTIONS
# ARM assembly is supported but experimental, disable it for the release
CMAKE_EXTRA_OPTIONS[arm-linux-gnueabihf]="-DSECP256K1_USE_ASM=OFF"
FAKETIME_HOST_PROGS=""
FAKETIME_PROGS="date ar ranlib nm"
HOST_CFLAGS="-O2 -g"
HOST_CXXFLAGS="-O2 -g"
HOST_LDFLAGS=-static-libstdc++
export QT_RCC_TEST=1
export QT_RCC_SOURCE_DATE_OVERRIDE=1
export TZ="UTC"
export BUILD_DIR=`pwd`
mkdir -p ${WRAP_DIR}
if test -n "$GBUILD_CACHE_ENABLED"; then
export SOURCES_PATH=${GBUILD_COMMON_CACHE}
export BASE_CACHE=${GBUILD_PACKAGE_CACHE}
mkdir -p ${BASE_CACHE} ${SOURCES_PATH}
fi
function create_global_faketime_wrappers {
for prog in ${FAKETIME_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
function create_per-host_faketime_wrappers {
for i in ${HOSTS[@]}; do
for prog in ${FAKETIME_HOST_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
}
function create_per-host_compiler_wrapper {
for i in ${HOSTS[@]}; do
for prog in gcc g++; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
echo "REAL=\`which -a ${i}-${prog}-9 | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
}
function create_native_compiler_wrapper {
for prog in gcc g++; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
echo "REAL=\`which -a x86_64-linux-gnu-${prog}-9 | head -1\`" >> ${WRAP_DIR}/${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
+ pip3 install lief==0.11.4
+
# Faketime for depends so intermediate results are comparable
export PATH_orig=${PATH}
create_global_faketime_wrappers "2000-01-01 12:00:00"
create_per-host_faketime_wrappers "2000-01-01 12:00:00"
# Wrap the compiler <host>-gcc-9 and <host>-g++-9 into <host>-gcc and
# <host>-g++
create_per-host_compiler_wrapper
# For the current host platform also wrap into regular gcc and g++, assume
# x86_64
create_native_compiler_wrapper
export PATH=${WRAP_DIR}:${PATH}
cd bitcoin
SOURCEDIR=`pwd`
BASEPREFIX=`pwd`/depends
# Build dependencies for each host
for i in ${HOSTS[@]}; do
make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}"
done
# Faketime for binaries
export PATH=${PATH_orig}
create_global_faketime_wrappers "${REFERENCE_DATETIME}"
create_per-host_faketime_wrappers "${REFERENCE_DATETIME}"
export PATH=${WRAP_DIR}:${PATH}
mkdir -p source_package
pushd source_package
# Any toolchain file will work for building the source package, just pick the
# first one
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${HOSTS[0]}]}
ninja package_source
SOURCEDIST=`echo bitcoin-abc-*.tar.gz`
mv ${SOURCEDIST} ..
popd
DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
# Correct tar file order
mkdir -p temp
pushd temp
tar -xf ../$SOURCEDIST
find bitcoin-abc-* | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST
popd
ORIGPATH="$PATH"
# Extract the release tarball into a dir for each host and build
for i in ${HOSTS[@]}; do
export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
mkdir -p distsrc-${i}
cd distsrc-${i}
INSTALLPATH=`pwd`/installed/${DISTNAME}
mkdir -p ${INSTALLPATH}
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${i}]} \
-DCLIENT_VERSION_IS_RELEASE=ON \
-DENABLE_CLANG_TIDY=OFF \
-DENABLE_REDUCE_EXPORTS=ON \
-DENABLE_STATIC_LIBSTDCXX=ON \
-DENABLE_GLIBC_BACK_COMPAT=ON \
-DCMAKE_INSTALL_PREFIX=${INSTALLPATH} \
-DCCACHE=OFF \
-DUSE_LINKER= \
${CMAKE_EXTRA_OPTIONS[${i}]}
ninja
ninja security-check
ninja symbol-check
ninja install-debug
cd installed
find ${DISTNAME} -not -name "*.dbg" | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz
find ${DISTNAME} -name "*.dbg" | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz
cd ../../
rm -rf distsrc-${i}
done
mkdir -p $OUTDIR/src
mv $SOURCEDIST $OUTDIR/src
diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml
index 5d067fdde..bafa05391 100644
--- a/contrib/gitian-descriptors/gitian-osx.yml
+++ b/contrib/gitian-descriptors/gitian-osx.yml
@@ -1,182 +1,185 @@
---
name: "bitcoin-abc-osx"
enable_cache: true
distro: "debian"
suites:
- "bullseye"
architectures:
- "amd64"
packages:
- "autoconf"
- "automake"
- "bsdmainutils"
- "ca-certificates"
- "cmake"
- "curl"
- "faketime"
- "fonts-tuffy"
- "g++"
- "git"
- "imagemagick"
- "libbz2-dev"
- "libcap-dev"
- "librsvg2-bin"
- "libtiff-tools"
- "libtinfo5"
- "libtool"
- "libz-dev"
- "ninja-build"
- "pkg-config"
- "python3"
- "python3-dev"
+- "python3-pip"
- "python3-setuptools"
remotes:
- "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git"
"dir": "bitcoin"
files:
- "Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz"
script: |
WRAP_DIR=$HOME/wrapped
HOSTS=(
x86_64-apple-darwin16
)
# CMake toolchain file name differ from host name
declare -A CMAKE_TOOLCHAIN_FILE
CMAKE_TOOLCHAIN_FILE[x86_64-apple-darwin16]=OSX.cmake
FAKETIME_HOST_PROGS=""
FAKETIME_PROGS="ar ranlib date dmg genisoimage"
export QT_RCC_TEST=1
export QT_RCC_SOURCE_DATE_OVERRIDE=1
export TZ="UTC"
export BUILD_DIR=`pwd`
mkdir -p ${WRAP_DIR}
if test -n "$GBUILD_CACHE_ENABLED"; then
export SOURCES_PATH=${GBUILD_COMMON_CACHE}
export BASE_CACHE=${GBUILD_PACKAGE_CACHE}
mkdir -p ${BASE_CACHE} ${SOURCES_PATH}
fi
export ZERO_AR_DATE=1
function create_global_faketime_wrappers {
for prog in ${FAKETIME_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
function create_per-host_faketime_wrappers {
for i in ${HOSTS[@]}; do
for prog in ${FAKETIME_HOST_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
}
+ pip3 install lief==0.11.4
+
# Faketime for depends so intermediate results are comparable
export PATH_orig=${PATH}
create_global_faketime_wrappers "2000-01-01 12:00:00"
create_per-host_faketime_wrappers "2000-01-01 12:00:00"
export PATH=${WRAP_DIR}:${PATH}
cd bitcoin
SOURCEDIR=`pwd`
BASEPREFIX=`pwd`/depends
mkdir -p ${BASEPREFIX}/SDKs
tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
# Build dependencies for each host
for i in ${HOSTS[@]}; do
make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}"
done
# Faketime for binaries
export PATH=${PATH_orig}
create_global_faketime_wrappers "${REFERENCE_DATETIME}"
create_per-host_faketime_wrappers "${REFERENCE_DATETIME}"
export PATH=${WRAP_DIR}:${PATH}
mkdir -p source_package
pushd source_package
# Any toolchain file will work for building the source package, just pick the
# first one
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${HOSTS[0]}]}
ninja package_source
SOURCEDIST=`echo bitcoin-abc-*.tar.gz`
mv ${SOURCEDIST} ..
popd
DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
# Correct tar file order
mkdir -p temp
pushd temp
tar -xf ../$SOURCEDIST
find bitcoin-abc-* | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST
popd
ORIGPATH="$PATH"
# Extract the release tarball into a dir for each host and build
for i in ${HOSTS[@]}; do
export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
mkdir -p distsrc-${i}
cd distsrc-${i}
INSTALLPATH=`pwd`/installed/${DISTNAME}
mkdir -p ${INSTALLPATH}
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${i}]} \
-DCLIENT_VERSION_IS_RELEASE=ON \
-DENABLE_CLANG_TIDY=OFF \
-DENABLE_REDUCE_EXPORTS=ON \
-DCMAKE_INSTALL_PREFIX=${INSTALLPATH} \
-DCCACHE=OFF \
-DGENISOIMAGE_EXECUTABLE="${WRAP_DIR}/genisoimage"
ninja
ninja security-check
ninja symbol-check
ninja install/strip
export PYTHONPATH="${BASEPREFIX}/${i}/native/lib/python3/dist-packages:${PYTHONPATH}"
ninja osx-deploydir
OSX_VOLNAME="$(cat osx_volname)"
mkdir -p unsigned-app-${i}
cp osx_volname unsigned-app-${i}/
cp ../contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i}
cp ../contrib/macdeploy/detached-sig-create.sh unsigned-app-${i}
cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i}
cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate
cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff
mv dist unsigned-app-${i}
pushd unsigned-app-${i}
find . | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz
popd
ninja osx-dmg
${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg
cd installed
find -path "*.app*" -type f -executable -exec mv {} ${DISTNAME}/bin/bitcoin-qt \;
find ${DISTNAME} -not -path "*.app*" | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz
cd ../../
done
mkdir -p $OUTDIR/src
mv $SOURCEDIST $OUTDIR/src
mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml
index 97a925340..e472dcb0c 100644
--- a/contrib/gitian-descriptors/gitian-win.yml
+++ b/contrib/gitian-descriptors/gitian-win.yml
@@ -1,185 +1,188 @@
---
name: "bitcoin-abc-win"
enable_cache: true
distro: "debian"
suites:
- "bullseye"
architectures:
- "amd64"
packages:
- "autoconf"
- "automake"
- "bsdmainutils"
- "ca-certificates"
- "cmake"
- "curl"
- "faketime"
- "g++"
- "g++-mingw-w64"
- "git"
- "libtool"
- "ninja-build"
- "mingw-w64"
- "nsis"
- "pkg-config"
- "python3"
+- "python3-pip"
- "zip"
remotes:
- "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git"
"dir": "bitcoin"
files: []
script: |
WRAP_DIR=$HOME/wrapped
HOSTS=(
x86_64-w64-mingw32
)
# CMake toolchain file name differ from host name
declare -A CMAKE_TOOLCHAIN_FILE
CMAKE_TOOLCHAIN_FILE[x86_64-w64-mingw32]=Win64.cmake
FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy"
FAKETIME_PROGS="date makensis zip"
HOST_CFLAGS="-O2 -g"
HOST_CXXFLAGS="-O2 -g"
export QT_RCC_TEST=1
export QT_RCC_SOURCE_DATE_OVERRIDE=1
export TZ="UTC"
export BUILD_DIR=`pwd`
mkdir -p ${WRAP_DIR}
if test -n "$GBUILD_CACHE_ENABLED"; then
export SOURCES_PATH=${GBUILD_COMMON_CACHE}
export BASE_CACHE=${GBUILD_PACKAGE_CACHE}
mkdir -p ${BASE_CACHE} ${SOURCES_PATH}
fi
function create_global_faketime_wrappers {
for prog in ${FAKETIME_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
function create_per-host_faketime_wrappers {
for i in ${HOSTS[@]}; do
for prog in ${FAKETIME_HOST_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
}
function create_per-host_compiler_wrapper {
# -posix variant is required for c++11 threading.
for i in ${HOSTS[@]}; do
mkdir -p ${WRAP_DIR}/${i}
for prog in gcc g++; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
}
+ pip3 install lief==0.11.4
+
# Faketime for depends so intermediate results are comparable
export PATH_orig=${PATH}
create_global_faketime_wrappers "2000-01-01 12:00:00"
create_per-host_faketime_wrappers "2000-01-01 12:00:00"
create_per-host_compiler_wrapper "2000-01-01 12:00:00"
export PATH=${WRAP_DIR}:${PATH}
cd bitcoin
SOURCEDIR=`pwd`
BASEPREFIX=`pwd`/depends
# Build dependencies for each host
for i in ${HOSTS[@]}; do
make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}"
done
# Faketime for binaries
export PATH=${PATH_orig}
create_global_faketime_wrappers "${REFERENCE_DATETIME}"
create_per-host_faketime_wrappers "${REFERENCE_DATETIME}"
create_per-host_compiler_wrapper "${REFERENCE_DATETIME}"
export PATH=${WRAP_DIR}:${PATH}
mkdir -p source_package
pushd source_package
# Any toolchain file will work for building the source package, just pick the
# first one
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${HOSTS[0]}]}
ninja package_source
SOURCEDIST=`echo bitcoin-abc-*.tar.gz`
mv ${SOURCEDIST} ..
popd
DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
# Correct tar file order
mkdir -p temp
pushd temp
tar -xf ../$SOURCEDIST
find bitcoin-abc-* | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST
mkdir -p $OUTDIR/src
cp ../$SOURCEDIST $OUTDIR/src
popd
# Allow extra cmake option to be specified for each host
declare -A CMAKE_EXTRA_OPTIONS
CMAKE_EXTRA_OPTIONS[x86_64-w64-mingw32]="-DCPACK_PACKAGE_FILE_NAME=${DISTNAME}-win64-setup-unsigned"
ORIGPATH="$PATH"
# Extract the release tarball into a dir for each host and build
for i in ${HOSTS[@]}; do
export PATH=${BASEPREFIX}/${i}/native/bin:${WRAP_DIR}:${ORIGPATH}
mkdir -p distsrc-${i}
cd distsrc-${i}
INSTALLPATH=`pwd`/installed/${DISTNAME}
mkdir -p ${INSTALLPATH}
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${i}]} \
-DCLIENT_VERSION_IS_RELEASE=ON \
-DENABLE_CLANG_TIDY=OFF \
-DENABLE_REDUCE_EXPORTS=ON \
-DBUILD_BITCOIN_SEEDER=OFF \
-DCPACK_STRIP_FILES=ON \
-DCMAKE_INSTALL_PREFIX=${INSTALLPATH} \
-DCCACHE=OFF \
${CMAKE_EXTRA_OPTIONS[${i}]}
ninja
ninja security-check
ninja symbol-check
ninja install-debug
ninja package
cp -f bitcoin-abc-*-setup-unsigned.exe ${OUTDIR}/
cd installed
mkdir -p ${DISTNAME}/lib
mv ${DISTNAME}/bin/*.dll* ${DISTNAME}/lib/
find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip
find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}-debug.zip
cd ../../
rm -rf distsrc-${i}
done
cd $OUTDIR
mv ${OUTDIR}/${DISTNAME}-x86_64-*-debug.zip ${OUTDIR}/${DISTNAME}-win64-debug.zip
mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip
diff --git a/contrib/utils/install-dependencies-bullseye.sh b/contrib/utils/install-dependencies-bullseye.sh
index 60f5fda5b..04af7afe0 100755
--- a/contrib/utils/install-dependencies-bullseye.sh
+++ b/contrib/utils/install-dependencies-bullseye.sh
@@ -1,182 +1,184 @@
#!/usr/bin/env bash
export LC_ALL=C.UTF-8
set -euxo pipefail
dpkg --add-architecture i386
PACKAGES=(
arcanist
automake
autotools-dev
binutils
bsdmainutils
build-essential
ccache
cmake
curl
default-jdk
devscripts
doxygen
dput
flake8
g++-9
g++-9-aarch64-linux-gnu
g++-9-arm-linux-gnueabihf
g++-9-multilib
g++-mingw-w64
gcc-9
gcc-9-aarch64-linux-gnu
gcc-9-arm-linux-gnueabihf
gcc-9-multilib
gettext-base
git
golang
gnupg
graphviz
gperf
help2man
imagemagick
jq
lcov
less
lib32stdc++-10-dev
libboost-all-dev
libbz2-dev
libc6-dev:i386
libcap-dev
libdb++-dev
libdb-dev
libevent-dev
libjemalloc-dev
libminiupnpc-dev
libnatpmp-dev
libprotobuf-dev
libqrencode-dev
libqt5core5a
libqt5dbus5
libqt5gui5
librsvg2-bin
libsqlite3-dev
libssl-dev
libtiff-tools
libtinfo5
libtool
libzmq3-dev
lld
make
ninja-build
nsis
pandoc
php-codesniffer
pkg-config
protobuf-compiler
python3
python3-autopep8
python3-pip
python3-setuptools
python3-yaml
python3-zmq
qemu-user-static
qttools5-dev
qttools5-dev-tools
shellcheck
software-properties-common
tar
wget
xvfb
yamllint
wine
)
function join_by() {
local IFS="$1"
shift
echo "$*"
}
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${PACKAGES[@]}")
# Install llvm and clang
apt-key add "$(dirname "$0")"/llvm.pub
add-apt-repository "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-12 main"
apt-get update
LLVM_PACKAGES=(
clang-12
clang-format-12
clang-tidy-12
clang-tools-12
)
DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${LLVM_PACKAGES[@]}")
# Make sure our specific llvm and clang versions have highest priority
update-alternatives --install /usr/bin/clang clang "$(command -v clang-12)" 100
update-alternatives --install /usr/bin/clang++ clang++ "$(command -v clang++-12)" 100
update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "$(command -v llvm-symbolizer-12)" 100
# Use gcc-9/g++-9 by default so it uses libstdc++-9. This prevents from pulling
# the new pthread_cond_clockwait symbol from GLIBC_30 and ensure we are testing
# under the same condition our release it built.
update-alternatives --install /usr/bin/gcc gcc "$(command -v gcc-9)" 100
update-alternatives --install /usr/bin/g++ g++ "$(command -v g++-9)" 100
update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc "$(command -v aarch64-linux-gnu-gcc-9)" 100
update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ "$(command -v aarch64-linux-gnu-g++-9)" 100
update-alternatives --install /usr/bin/arm-linux-gnueabihf-gcc arm-linux-gnueabihf-gcc "$(command -v arm-linux-gnueabihf-gcc-9)" 100
update-alternatives --install /usr/bin/arm-linux-gnueabihf-g++ arm-linux-gnueabihf-g++ "$(command -v arm-linux-gnueabihf-g++-9)" 100
# Use the mingw posix variant
update-alternatives --set x86_64-w64-mingw32-g++ $(command -v x86_64-w64-mingw32-g++-posix)
update-alternatives --set x86_64-w64-mingw32-gcc $(command -v x86_64-w64-mingw32-gcc-posix)
# Python library for merging nested structures
pip3 install deepmerge
# For running Python test suites
pip3 install pytest
# For en/-decoding protobuf messages
# This version is compatible with Debian's "protobuf-compiler" package
pip3 install "protobuf<=3.20"
+# For security-check.py and symbol-check.py
+pip3 install "lief>=0.11.4"
# Up-to-date mypy, isort and flynt packages are required python linters
pip3 install isort==5.6.4 mypy==0.780 flynt==0.78
echo "export PATH=\"$(python3 -m site --user-base)/bin:\$PATH\"" >> ~/.bashrc
# shellcheck source=/dev/null
source ~/.bashrc
# Install npm v7.x and nodejs v15.x
curl -sL https://deb.nodesource.com/setup_15.x | bash -
apt-get install -y nodejs
# Install nyc for mocha unit test reporting
npm i -g nyc
# Install Rust stable 1.67.1 and nightly from the 2023-02-17
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=1.67.1
RUST_HOME="${HOME}/.cargo/bin"
RUST_NIGHTLY_DATE=2023-02-17
"${RUST_HOME}/rustup" install nightly-${RUST_NIGHTLY_DATE}
"${RUST_HOME}/rustup" component add rustfmt --toolchain nightly-${RUST_NIGHTLY_DATE}
# Name the nightly toolchain "abc-nightly"
"${RUST_HOME}/rustup" toolchain link abc-nightly "$(${RUST_HOME}/rustc +nightly-${RUST_NIGHTLY_DATE} --print sysroot)"
# Install required compile platform targets on stable
"${RUST_HOME}/rustup" target add "i686-unknown-linux-gnu" \
"x86_64-unknown-linux-gnu" \
"aarch64-unknown-linux-gnu" \
"arm-unknown-linux-gnueabihf" \
"x86_64-apple-darwin" \
"x86_64-pc-windows-gnu"
# Install corrosion from Github
wget https://api.github.com/repos/corrosion-rs/corrosion/tarball/v0.3.0 -O corrosion.tar.gz
echo "3b9a48737264add649983df26c83f3285ce17e20d86194c7756689a0d8470267 corrosion.tar.gz" | sha256sum -c
tar xzf corrosion.tar.gz
CORROSION_SRC_FOLDER=corrosion-rs-corrosion-b764a9f
CORROSION_BUILD_FOLDER=${CORROSION_SRC_FOLDER}-build
cmake -S${CORROSION_SRC_FOLDER} -B${CORROSION_BUILD_FOLDER} -DCMAKE_BUILD_TYPE=Release
cmake --build ${CORROSION_BUILD_FOLDER} --config Release
cmake --install ${CORROSION_BUILD_FOLDER} --config Release

File Metadata

Mime Type
text/x-diff
Expires
Sat, Nov 23, 09:56 (1 d, 16 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
4559199
Default Alt Text
(37 KB)

Event Timeline