Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Files
F10615097
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
37 KB
Subscribers
None
View Options
diff --git a/cmake/modules/BinaryTest.cmake b/cmake/modules/BinaryTest.cmake
index 73cb9492b..9553bab6a 100644
--- a/cmake/modules/BinaryTest.cmake
+++ b/cmake/modules/BinaryTest.cmake
@@ -1,41 +1,40 @@
# Facilities to run tests on the executable binaries.
macro(add_to_symbols_check BINARY)
if(NOT TARGET symbol-check)
add_custom_target(symbol-check)
endif()
set(CUSTOM_TARGET_NAME "symbol-check-${BINARY}")
add_custom_target("${CUSTOM_TARGET_NAME}"
COMMENT "Running symbol-check.py on ${BINARY}..."
COMMAND
"OTOOL=${CMAKE_OTOOL}"
"${Python_EXECUTABLE}"
"${CMAKE_SOURCE_DIR}/contrib/devtools/symbol-check.py"
"$<TARGET_FILE:${BINARY}>"
DEPENDS
"${BINARY}"
)
add_dependencies(symbol-check "${CUSTOM_TARGET_NAME}")
endmacro()
macro(add_to_security_check BINARY)
if(NOT TARGET security-check)
add_custom_target(security-check)
endif()
set(CUSTOM_TARGET_NAME "security-check-${BINARY}")
add_custom_target("${CUSTOM_TARGET_NAME}"
COMMENT "Running security-check.py on ${BINARY}..."
COMMAND
- "OTOOL=${CMAKE_OTOOL}"
"${Python_EXECUTABLE}"
"${CMAKE_SOURCE_DIR}/contrib/devtools/security-check.py"
"$<TARGET_FILE:${BINARY}>"
DEPENDS
"${BINARY}"
)
add_dependencies(security-check "${CUSTOM_TARGET_NAME}")
endmacro()
diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py
index 4b4c06f32..8dc5b44b7 100755
--- a/contrib/devtools/security-check.py
+++ b/contrib/devtools/security-check.py
@@ -1,312 +1,255 @@
#!/usr/bin/env python3
# Copyright (c) 2015-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
'''
Perform basic security checks on a series of executables.
Exit status will be 0 if successful, and the program will be silent.
Otherwise the exit status will be 1 and it will log which executables failed which checks.
-Needs `objdump` (for PE) and `otool` (for MACHO).
'''
-import subprocess
import sys
-import os
from typing import List, Optional
+import lief
import pixie
-OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump')
-OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool')
-
-
-def run_command(command) -> str:
- p = subprocess.run(
- command,
- stdout=subprocess.PIPE,
- check=True,
- universal_newlines=True)
- return p.stdout
-
def check_ELF_PIE(executable) -> bool:
'''
Check for position independent executable (PIE), allowing for address space
randomization.
'''
elf = pixie.load(executable)
return elf.hdr.e_type == pixie.ET_DYN
def check_ELF_NX(executable) -> bool:
'''
Check that no sections are writable and executable (including the stack)
'''
elf = pixie.load(executable)
have_wx = False
have_gnu_stack = False
for ph in elf.program_headers:
if ph.p_type == pixie.PT_GNU_STACK:
have_gnu_stack = True
# section is both writable and executable
if (ph.p_flags & pixie.PF_W) != 0 and (ph.p_flags & pixie.PF_X) != 0:
have_wx = True
return have_gnu_stack and not have_wx
def check_ELF_RELRO(executable) -> bool:
'''
Check for read-only relocations.
GNU_RELRO program header must exist
Dynamic section must have BIND_NOW flag
'''
elf = pixie.load(executable)
have_gnu_relro = False
for ph in elf.program_headers:
# Note: not checking p_flags == PF_R: here as linkers set the permission
# differently. This does not affect security: the permission flags of the
# GNU_RELRO program header are ignored, the PT_LOAD header determines the
# effective permissions.
# However, the dynamic linker need to write to this area so these are RW.
# Glibc itself takes care of mprotecting this area R after relocations are
# finished. See also https://marc.info/?l=binutils&m=1498883354122353
if ph.p_type == pixie.PT_GNU_RELRO:
have_gnu_relro = True
have_bindnow = False
for flags in elf.query_dyn_tags(pixie.DT_FLAGS):
assert isinstance(flags, int)
if flags & pixie.DF_BIND_NOW:
have_bindnow = True
return have_gnu_relro and have_bindnow
def check_ELF_Canary(executable) -> bool:
'''
Check for use of stack canary
'''
elf = pixie.load(executable)
for symbol in elf.dyn_symbols:
if symbol.name == b'__stack_chk_fail':
return True
return False
def check_ELF_separate_code(executable):
'''
Check that sections are appropriately separated in virtual memory,
based on their permissions. This checks for missing -Wl,-z,separate-code
and potentially other problems.
'''
elf = pixie.load(executable)
R = pixie.PF_R
W = pixie.PF_W
E = pixie.PF_X
EXPECTED_FLAGS = {
# Read + execute
b'.init': R | E,
b'.plt': R | E,
b'.plt.got': R | E,
b'.plt.sec': R | E,
b'.text': R | E,
b'.fini': R | E,
# Read-only data
b'.interp': R,
b'.note.gnu.property': R,
b'.note.gnu.build-id': R,
b'.note.ABI-tag': R,
b'.gnu.hash': R,
b'.dynsym': R,
b'.dynstr': R,
b'.gnu.version': R,
b'.gnu.version_r': R,
b'.rela.dyn': R,
b'.rela.plt': R,
b'.rodata': R,
b'.eh_frame_hdr': R,
b'.eh_frame': R,
b'.qtmetadata': R,
b'.gcc_except_table': R,
b'.stapsdt.base': R,
# Writable data
b'.init_array': R | W,
b'.fini_array': R | W,
b'.dynamic': R | W,
b'.got': R | W,
b'.data': R | W,
b'.bss': R | W,
}
if elf.hdr.e_machine == pixie.EM_PPC64:
# .plt is RW on ppc64 even with separate-code
EXPECTED_FLAGS[b'.plt'] = R | W
# For all LOAD program headers get mapping to the list of sections,
# and for each section, remember the flags of the associated program header.
flags_per_section = {}
for ph in elf.program_headers:
if ph.p_type == pixie.PT_LOAD:
for section in ph.sections:
assert(section.name not in flags_per_section)
flags_per_section[section.name] = ph.p_flags
# Spot-check ELF LOAD program header flags per section
# If these sections exist, check them against the expected R/W/E flags
for (section, flags) in flags_per_section.items():
if section in EXPECTED_FLAGS:
if EXPECTED_FLAGS[section] != flags:
return False
return True
-def get_PE_dll_characteristics(executable) -> int:
- '''Get PE DllCharacteristics bits'''
- stdout = run_command([OBJDUMP_CMD, '-x', executable])
-
- bits = 0
- for line in stdout.splitlines():
- tokens = line.split()
- if len(tokens) >= 2 and tokens[0] == 'DllCharacteristics':
- bits = int(tokens[1], 16)
- return bits
-
-
-IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020
-IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE = 0x0040
-IMAGE_DLL_CHARACTERISTICS_NX_COMPAT = 0x0100
-
-
def check_PE_DYNAMIC_BASE(executable) -> bool:
'''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)'''
- bits = get_PE_dll_characteristics(executable)
- return (bits & IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE) == IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE
+ binary = lief.parse(executable)
+ return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists
# Must support high-entropy 64-bit address space layout randomization
# in addition to DYNAMIC_BASE to have secure ASLR.
def check_PE_HIGH_ENTROPY_VA(executable) -> bool:
'''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR'''
- bits = get_PE_dll_characteristics(executable)
- return (bits & IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA) == IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA
+ binary = lief.parse(executable)
+ return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists
def check_PE_RELOC_SECTION(executable) -> bool:
'''Check for a reloc section. This is required for functional ASLR.'''
- stdout = run_command([OBJDUMP_CMD, '-h', executable])
-
- for line in stdout.splitlines():
- if '.reloc' in line:
- return True
- return False
-
-
-def check_PE_NX(executable) -> bool:
- '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)'''
- bits = get_PE_dll_characteristics(executable)
- return (
- bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT
-
+ binary = lief.parse(executable)
+ return binary.has_relocations
-def get_MACHO_executable_flags(executable) -> List[str]:
- stdout = run_command([OTOOL_CMD, '-vh', executable])
- flags: List[str] = []
- for line in stdout.splitlines():
- tokens = line.split()
- # filter first two header lines
- if 'magic' in tokens or 'Mach' in tokens:
- continue
- # filter ncmds and sizeofcmds values
- flags += [t for t in tokens if not t.isdigit()]
- return flags
-
-
-def check_MACHO_PIE(executable) -> bool:
+def check_MACHO_NOUNDEFS(executable) -> bool:
'''
- Check for position independent executable (PIE), allowing for address space randomization.
+ Check for no undefined references.
'''
- flags = get_MACHO_executable_flags(executable)
- return 'PIE' in flags
+ binary = lief.parse(executable)
+ return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS)
-def check_MACHO_NOUNDEFS(executable) -> bool:
+def check_MACHO_Canary(executable) -> bool:
'''
- Check for no undefined references.
+ Check for use of stack canary
'''
- flags = get_MACHO_executable_flags(executable)
- return 'NOUNDEFS' in flags
+ binary = lief.parse(executable)
+ return binary.has_symbol('___stack_chk_fail')
-def check_MACHO_NX(executable) -> bool:
+def check_PIE(executable) -> bool:
'''
- Check for no stack execution
+ Check for position independent executable (PIE),
+ allowing for address space randomization.
'''
- flags = get_MACHO_executable_flags(executable)
- return 'ALLOW_STACK_EXECUTION' not in flags
+ binary = lief.parse(executable)
+ return binary.is_pie
-def check_MACHO_Canary(executable) -> bool:
+def check_NX(executable) -> bool:
'''
- Check for use of stack canary
+ Check for no stack execution
'''
- stdout = run_command([OTOOL_CMD, '-Iv', executable])
- for line in stdout.splitlines():
- if '___stack_chk_fail' in line:
- return True
- return False
+ binary = lief.parse(executable)
+ return binary.has_nx
CHECKS = {
'ELF': [
('PIE', check_ELF_PIE),
('NX', check_ELF_NX),
('RELRO', check_ELF_RELRO),
('Canary', check_ELF_Canary),
('separate_code', check_ELF_separate_code),
],
'PE': [
+ ('PIE', check_PIE),
('DYNAMIC_BASE', check_PE_DYNAMIC_BASE),
('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA),
- ('NX', check_PE_NX),
+ ('NX', check_NX),
('RELOC_SECTION', check_PE_RELOC_SECTION),
],
'MACHO': [
- ('PIE', check_MACHO_PIE),
+ ('PIE', check_PIE),
('NOUNDEFS', check_MACHO_NOUNDEFS),
- ('NX', check_MACHO_NX),
+ ('NX', check_NX),
('Canary', check_MACHO_Canary),
]
}
def identify_executable(executable) -> Optional[str]:
with open(filename, 'rb') as f:
magic = f.read(4)
if magic.startswith(b'MZ'):
return 'PE'
elif magic.startswith(b'\x7fELF'):
return 'ELF'
elif magic.startswith(b'\xcf\xfa'):
return 'MACHO'
return None
if __name__ == '__main__':
- retval = 0
+ retval: int = 0
for filename in sys.argv[1:]:
try:
etype = identify_executable(filename)
if etype is None:
- print('{}: unknown format'.format(filename))
+ print(f'{filename}: unknown format')
retval = 1
continue
- failed = []
+ failed: List[str] = []
for (name, func) in CHECKS[etype]:
if not func(filename):
failed.append(name)
if failed:
- print('{}: failed {}'.format(filename, ' '.join(failed)))
+ print(f"{filename}: failed {' '.join(failed)}")
retval = 1
except IOError:
- print('{}: cannot open'.format(filename))
+ print(f'{filename}: cannot open')
retval = 1
sys.exit(retval)
diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml
index 393992a04..f9dd3485a 100644
--- a/contrib/gitian-descriptors/gitian-linux.yml
+++ b/contrib/gitian-descriptors/gitian-linux.yml
@@ -1,201 +1,204 @@
---
name: "bitcoin-abc-linux"
enable_cache: true
distro: "debian"
suites:
- "bullseye"
architectures:
- "amd64"
packages:
- "autoconf"
- "automake"
- "binutils-aarch64-linux-gnu"
- "binutils-arm-linux-gnueabihf"
- "binutils-gold"
- "bsdmainutils"
- "ca-certificates"
- "cmake"
- "curl"
- "faketime"
# Use gcc/g++ 9 to avoid introducing the new pthread_cond_clockwait from glibc
# 2.30, which would make our release binary incompatible with systems using an
# older glibc version.
- "g++-9"
- "g++-9-aarch64-linux-gnu"
- "g++-9-arm-linux-gnueabihf"
- "gcc-9"
- "gcc-9-aarch64-linux-gnu"
- "gcc-9-arm-linux-gnueabihf"
- "git"
- "gperf"
- "libtool"
- "ninja-build"
- "pkg-config"
- "python3"
+- "python3-pip"
remotes:
- "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git"
"dir": "bitcoin"
files: []
script: |
WRAP_DIR=$HOME/wrapped
HOSTS=(
x86_64-linux-gnu
arm-linux-gnueabihf
aarch64-linux-gnu
)
# CMake toolchain file name differ from host name
declare -A CMAKE_TOOLCHAIN_FILE
CMAKE_TOOLCHAIN_FILE[x86_64-linux-gnu]=Linux64.cmake
CMAKE_TOOLCHAIN_FILE[arm-linux-gnueabihf]=LinuxARM.cmake
CMAKE_TOOLCHAIN_FILE[aarch64-linux-gnu]=LinuxAArch64.cmake
# Allow extra cmake option to be specified for each host
declare -A CMAKE_EXTRA_OPTIONS
# ARM assembly is supported but experimental, disable it for the release
CMAKE_EXTRA_OPTIONS[arm-linux-gnueabihf]="-DSECP256K1_USE_ASM=OFF"
FAKETIME_HOST_PROGS=""
FAKETIME_PROGS="date ar ranlib nm"
HOST_CFLAGS="-O2 -g"
HOST_CXXFLAGS="-O2 -g"
HOST_LDFLAGS=-static-libstdc++
export QT_RCC_TEST=1
export QT_RCC_SOURCE_DATE_OVERRIDE=1
export TZ="UTC"
export BUILD_DIR=`pwd`
mkdir -p ${WRAP_DIR}
if test -n "$GBUILD_CACHE_ENABLED"; then
export SOURCES_PATH=${GBUILD_COMMON_CACHE}
export BASE_CACHE=${GBUILD_PACKAGE_CACHE}
mkdir -p ${BASE_CACHE} ${SOURCES_PATH}
fi
function create_global_faketime_wrappers {
for prog in ${FAKETIME_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
function create_per-host_faketime_wrappers {
for i in ${HOSTS[@]}; do
for prog in ${FAKETIME_HOST_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
}
function create_per-host_compiler_wrapper {
for i in ${HOSTS[@]}; do
for prog in gcc g++; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
echo "REAL=\`which -a ${i}-${prog}-9 | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
}
function create_native_compiler_wrapper {
for prog in gcc g++; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
echo "REAL=\`which -a x86_64-linux-gnu-${prog}-9 | head -1\`" >> ${WRAP_DIR}/${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
+ pip3 install lief==0.11.4
+
# Faketime for depends so intermediate results are comparable
export PATH_orig=${PATH}
create_global_faketime_wrappers "2000-01-01 12:00:00"
create_per-host_faketime_wrappers "2000-01-01 12:00:00"
# Wrap the compiler <host>-gcc-9 and <host>-g++-9 into <host>-gcc and
# <host>-g++
create_per-host_compiler_wrapper
# For the current host platform also wrap into regular gcc and g++, assume
# x86_64
create_native_compiler_wrapper
export PATH=${WRAP_DIR}:${PATH}
cd bitcoin
SOURCEDIR=`pwd`
BASEPREFIX=`pwd`/depends
# Build dependencies for each host
for i in ${HOSTS[@]}; do
make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}"
done
# Faketime for binaries
export PATH=${PATH_orig}
create_global_faketime_wrappers "${REFERENCE_DATETIME}"
create_per-host_faketime_wrappers "${REFERENCE_DATETIME}"
export PATH=${WRAP_DIR}:${PATH}
mkdir -p source_package
pushd source_package
# Any toolchain file will work for building the source package, just pick the
# first one
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${HOSTS[0]}]}
ninja package_source
SOURCEDIST=`echo bitcoin-abc-*.tar.gz`
mv ${SOURCEDIST} ..
popd
DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
# Correct tar file order
mkdir -p temp
pushd temp
tar -xf ../$SOURCEDIST
find bitcoin-abc-* | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST
popd
ORIGPATH="$PATH"
# Extract the release tarball into a dir for each host and build
for i in ${HOSTS[@]}; do
export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
mkdir -p distsrc-${i}
cd distsrc-${i}
INSTALLPATH=`pwd`/installed/${DISTNAME}
mkdir -p ${INSTALLPATH}
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${i}]} \
-DCLIENT_VERSION_IS_RELEASE=ON \
-DENABLE_CLANG_TIDY=OFF \
-DENABLE_REDUCE_EXPORTS=ON \
-DENABLE_STATIC_LIBSTDCXX=ON \
-DENABLE_GLIBC_BACK_COMPAT=ON \
-DCMAKE_INSTALL_PREFIX=${INSTALLPATH} \
-DCCACHE=OFF \
-DUSE_LINKER= \
${CMAKE_EXTRA_OPTIONS[${i}]}
ninja
ninja security-check
ninja symbol-check
ninja install-debug
cd installed
find ${DISTNAME} -not -name "*.dbg" | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz
find ${DISTNAME} -name "*.dbg" | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz
cd ../../
rm -rf distsrc-${i}
done
mkdir -p $OUTDIR/src
mv $SOURCEDIST $OUTDIR/src
diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml
index 5d067fdde..bafa05391 100644
--- a/contrib/gitian-descriptors/gitian-osx.yml
+++ b/contrib/gitian-descriptors/gitian-osx.yml
@@ -1,182 +1,185 @@
---
name: "bitcoin-abc-osx"
enable_cache: true
distro: "debian"
suites:
- "bullseye"
architectures:
- "amd64"
packages:
- "autoconf"
- "automake"
- "bsdmainutils"
- "ca-certificates"
- "cmake"
- "curl"
- "faketime"
- "fonts-tuffy"
- "g++"
- "git"
- "imagemagick"
- "libbz2-dev"
- "libcap-dev"
- "librsvg2-bin"
- "libtiff-tools"
- "libtinfo5"
- "libtool"
- "libz-dev"
- "ninja-build"
- "pkg-config"
- "python3"
- "python3-dev"
+- "python3-pip"
- "python3-setuptools"
remotes:
- "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git"
"dir": "bitcoin"
files:
- "Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz"
script: |
WRAP_DIR=$HOME/wrapped
HOSTS=(
x86_64-apple-darwin16
)
# CMake toolchain file name differ from host name
declare -A CMAKE_TOOLCHAIN_FILE
CMAKE_TOOLCHAIN_FILE[x86_64-apple-darwin16]=OSX.cmake
FAKETIME_HOST_PROGS=""
FAKETIME_PROGS="ar ranlib date dmg genisoimage"
export QT_RCC_TEST=1
export QT_RCC_SOURCE_DATE_OVERRIDE=1
export TZ="UTC"
export BUILD_DIR=`pwd`
mkdir -p ${WRAP_DIR}
if test -n "$GBUILD_CACHE_ENABLED"; then
export SOURCES_PATH=${GBUILD_COMMON_CACHE}
export BASE_CACHE=${GBUILD_PACKAGE_CACHE}
mkdir -p ${BASE_CACHE} ${SOURCES_PATH}
fi
export ZERO_AR_DATE=1
function create_global_faketime_wrappers {
for prog in ${FAKETIME_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
function create_per-host_faketime_wrappers {
for i in ${HOSTS[@]}; do
for prog in ${FAKETIME_HOST_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
}
+ pip3 install lief==0.11.4
+
# Faketime for depends so intermediate results are comparable
export PATH_orig=${PATH}
create_global_faketime_wrappers "2000-01-01 12:00:00"
create_per-host_faketime_wrappers "2000-01-01 12:00:00"
export PATH=${WRAP_DIR}:${PATH}
cd bitcoin
SOURCEDIR=`pwd`
BASEPREFIX=`pwd`/depends
mkdir -p ${BASEPREFIX}/SDKs
tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
# Build dependencies for each host
for i in ${HOSTS[@]}; do
make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}"
done
# Faketime for binaries
export PATH=${PATH_orig}
create_global_faketime_wrappers "${REFERENCE_DATETIME}"
create_per-host_faketime_wrappers "${REFERENCE_DATETIME}"
export PATH=${WRAP_DIR}:${PATH}
mkdir -p source_package
pushd source_package
# Any toolchain file will work for building the source package, just pick the
# first one
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${HOSTS[0]}]}
ninja package_source
SOURCEDIST=`echo bitcoin-abc-*.tar.gz`
mv ${SOURCEDIST} ..
popd
DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
# Correct tar file order
mkdir -p temp
pushd temp
tar -xf ../$SOURCEDIST
find bitcoin-abc-* | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST
popd
ORIGPATH="$PATH"
# Extract the release tarball into a dir for each host and build
for i in ${HOSTS[@]}; do
export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
mkdir -p distsrc-${i}
cd distsrc-${i}
INSTALLPATH=`pwd`/installed/${DISTNAME}
mkdir -p ${INSTALLPATH}
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${i}]} \
-DCLIENT_VERSION_IS_RELEASE=ON \
-DENABLE_CLANG_TIDY=OFF \
-DENABLE_REDUCE_EXPORTS=ON \
-DCMAKE_INSTALL_PREFIX=${INSTALLPATH} \
-DCCACHE=OFF \
-DGENISOIMAGE_EXECUTABLE="${WRAP_DIR}/genisoimage"
ninja
ninja security-check
ninja symbol-check
ninja install/strip
export PYTHONPATH="${BASEPREFIX}/${i}/native/lib/python3/dist-packages:${PYTHONPATH}"
ninja osx-deploydir
OSX_VOLNAME="$(cat osx_volname)"
mkdir -p unsigned-app-${i}
cp osx_volname unsigned-app-${i}/
cp ../contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i}
cp ../contrib/macdeploy/detached-sig-create.sh unsigned-app-${i}
cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i}
cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate
cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff
mv dist unsigned-app-${i}
pushd unsigned-app-${i}
find . | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz
popd
ninja osx-dmg
${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg
cd installed
find -path "*.app*" -type f -executable -exec mv {} ${DISTNAME}/bin/bitcoin-qt \;
find ${DISTNAME} -not -path "*.app*" | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz
cd ../../
done
mkdir -p $OUTDIR/src
mv $SOURCEDIST $OUTDIR/src
mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml
index 97a925340..e472dcb0c 100644
--- a/contrib/gitian-descriptors/gitian-win.yml
+++ b/contrib/gitian-descriptors/gitian-win.yml
@@ -1,185 +1,188 @@
---
name: "bitcoin-abc-win"
enable_cache: true
distro: "debian"
suites:
- "bullseye"
architectures:
- "amd64"
packages:
- "autoconf"
- "automake"
- "bsdmainutils"
- "ca-certificates"
- "cmake"
- "curl"
- "faketime"
- "g++"
- "g++-mingw-w64"
- "git"
- "libtool"
- "ninja-build"
- "mingw-w64"
- "nsis"
- "pkg-config"
- "python3"
+- "python3-pip"
- "zip"
remotes:
- "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git"
"dir": "bitcoin"
files: []
script: |
WRAP_DIR=$HOME/wrapped
HOSTS=(
x86_64-w64-mingw32
)
# CMake toolchain file name differ from host name
declare -A CMAKE_TOOLCHAIN_FILE
CMAKE_TOOLCHAIN_FILE[x86_64-w64-mingw32]=Win64.cmake
FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy"
FAKETIME_PROGS="date makensis zip"
HOST_CFLAGS="-O2 -g"
HOST_CXXFLAGS="-O2 -g"
export QT_RCC_TEST=1
export QT_RCC_SOURCE_DATE_OVERRIDE=1
export TZ="UTC"
export BUILD_DIR=`pwd`
mkdir -p ${WRAP_DIR}
if test -n "$GBUILD_CACHE_ENABLED"; then
export SOURCES_PATH=${GBUILD_COMMON_CACHE}
export BASE_CACHE=${GBUILD_PACKAGE_CACHE}
mkdir -p ${BASE_CACHE} ${SOURCES_PATH}
fi
function create_global_faketime_wrappers {
for prog in ${FAKETIME_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
function create_per-host_faketime_wrappers {
for i in ${HOSTS[@]}; do
for prog in ${FAKETIME_HOST_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
}
function create_per-host_compiler_wrapper {
# -posix variant is required for c++11 threading.
for i in ${HOSTS[@]}; do
mkdir -p ${WRAP_DIR}/${i}
for prog in gcc g++; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
}
+ pip3 install lief==0.11.4
+
# Faketime for depends so intermediate results are comparable
export PATH_orig=${PATH}
create_global_faketime_wrappers "2000-01-01 12:00:00"
create_per-host_faketime_wrappers "2000-01-01 12:00:00"
create_per-host_compiler_wrapper "2000-01-01 12:00:00"
export PATH=${WRAP_DIR}:${PATH}
cd bitcoin
SOURCEDIR=`pwd`
BASEPREFIX=`pwd`/depends
# Build dependencies for each host
for i in ${HOSTS[@]}; do
make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}"
done
# Faketime for binaries
export PATH=${PATH_orig}
create_global_faketime_wrappers "${REFERENCE_DATETIME}"
create_per-host_faketime_wrappers "${REFERENCE_DATETIME}"
create_per-host_compiler_wrapper "${REFERENCE_DATETIME}"
export PATH=${WRAP_DIR}:${PATH}
mkdir -p source_package
pushd source_package
# Any toolchain file will work for building the source package, just pick the
# first one
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${HOSTS[0]}]}
ninja package_source
SOURCEDIST=`echo bitcoin-abc-*.tar.gz`
mv ${SOURCEDIST} ..
popd
DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
# Correct tar file order
mkdir -p temp
pushd temp
tar -xf ../$SOURCEDIST
find bitcoin-abc-* | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST
mkdir -p $OUTDIR/src
cp ../$SOURCEDIST $OUTDIR/src
popd
# Allow extra cmake option to be specified for each host
declare -A CMAKE_EXTRA_OPTIONS
CMAKE_EXTRA_OPTIONS[x86_64-w64-mingw32]="-DCPACK_PACKAGE_FILE_NAME=${DISTNAME}-win64-setup-unsigned"
ORIGPATH="$PATH"
# Extract the release tarball into a dir for each host and build
for i in ${HOSTS[@]}; do
export PATH=${BASEPREFIX}/${i}/native/bin:${WRAP_DIR}:${ORIGPATH}
mkdir -p distsrc-${i}
cd distsrc-${i}
INSTALLPATH=`pwd`/installed/${DISTNAME}
mkdir -p ${INSTALLPATH}
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${i}]} \
-DCLIENT_VERSION_IS_RELEASE=ON \
-DENABLE_CLANG_TIDY=OFF \
-DENABLE_REDUCE_EXPORTS=ON \
-DBUILD_BITCOIN_SEEDER=OFF \
-DCPACK_STRIP_FILES=ON \
-DCMAKE_INSTALL_PREFIX=${INSTALLPATH} \
-DCCACHE=OFF \
${CMAKE_EXTRA_OPTIONS[${i}]}
ninja
ninja security-check
ninja symbol-check
ninja install-debug
ninja package
cp -f bitcoin-abc-*-setup-unsigned.exe ${OUTDIR}/
cd installed
mkdir -p ${DISTNAME}/lib
mv ${DISTNAME}/bin/*.dll* ${DISTNAME}/lib/
find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip
find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}-debug.zip
cd ../../
rm -rf distsrc-${i}
done
cd $OUTDIR
mv ${OUTDIR}/${DISTNAME}-x86_64-*-debug.zip ${OUTDIR}/${DISTNAME}-win64-debug.zip
mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip
diff --git a/contrib/utils/install-dependencies-bullseye.sh b/contrib/utils/install-dependencies-bullseye.sh
index 60f5fda5b..04af7afe0 100755
--- a/contrib/utils/install-dependencies-bullseye.sh
+++ b/contrib/utils/install-dependencies-bullseye.sh
@@ -1,182 +1,184 @@
#!/usr/bin/env bash
export LC_ALL=C.UTF-8
set -euxo pipefail
dpkg --add-architecture i386
PACKAGES=(
arcanist
automake
autotools-dev
binutils
bsdmainutils
build-essential
ccache
cmake
curl
default-jdk
devscripts
doxygen
dput
flake8
g++-9
g++-9-aarch64-linux-gnu
g++-9-arm-linux-gnueabihf
g++-9-multilib
g++-mingw-w64
gcc-9
gcc-9-aarch64-linux-gnu
gcc-9-arm-linux-gnueabihf
gcc-9-multilib
gettext-base
git
golang
gnupg
graphviz
gperf
help2man
imagemagick
jq
lcov
less
lib32stdc++-10-dev
libboost-all-dev
libbz2-dev
libc6-dev:i386
libcap-dev
libdb++-dev
libdb-dev
libevent-dev
libjemalloc-dev
libminiupnpc-dev
libnatpmp-dev
libprotobuf-dev
libqrencode-dev
libqt5core5a
libqt5dbus5
libqt5gui5
librsvg2-bin
libsqlite3-dev
libssl-dev
libtiff-tools
libtinfo5
libtool
libzmq3-dev
lld
make
ninja-build
nsis
pandoc
php-codesniffer
pkg-config
protobuf-compiler
python3
python3-autopep8
python3-pip
python3-setuptools
python3-yaml
python3-zmq
qemu-user-static
qttools5-dev
qttools5-dev-tools
shellcheck
software-properties-common
tar
wget
xvfb
yamllint
wine
)
function join_by() {
local IFS="$1"
shift
echo "$*"
}
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${PACKAGES[@]}")
# Install llvm and clang
apt-key add "$(dirname "$0")"/llvm.pub
add-apt-repository "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-12 main"
apt-get update
LLVM_PACKAGES=(
clang-12
clang-format-12
clang-tidy-12
clang-tools-12
)
DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${LLVM_PACKAGES[@]}")
# Make sure our specific llvm and clang versions have highest priority
update-alternatives --install /usr/bin/clang clang "$(command -v clang-12)" 100
update-alternatives --install /usr/bin/clang++ clang++ "$(command -v clang++-12)" 100
update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "$(command -v llvm-symbolizer-12)" 100
# Use gcc-9/g++-9 by default so it uses libstdc++-9. This prevents from pulling
# the new pthread_cond_clockwait symbol from GLIBC_30 and ensure we are testing
# under the same condition our release it built.
update-alternatives --install /usr/bin/gcc gcc "$(command -v gcc-9)" 100
update-alternatives --install /usr/bin/g++ g++ "$(command -v g++-9)" 100
update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc "$(command -v aarch64-linux-gnu-gcc-9)" 100
update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ "$(command -v aarch64-linux-gnu-g++-9)" 100
update-alternatives --install /usr/bin/arm-linux-gnueabihf-gcc arm-linux-gnueabihf-gcc "$(command -v arm-linux-gnueabihf-gcc-9)" 100
update-alternatives --install /usr/bin/arm-linux-gnueabihf-g++ arm-linux-gnueabihf-g++ "$(command -v arm-linux-gnueabihf-g++-9)" 100
# Use the mingw posix variant
update-alternatives --set x86_64-w64-mingw32-g++ $(command -v x86_64-w64-mingw32-g++-posix)
update-alternatives --set x86_64-w64-mingw32-gcc $(command -v x86_64-w64-mingw32-gcc-posix)
# Python library for merging nested structures
pip3 install deepmerge
# For running Python test suites
pip3 install pytest
# For en/-decoding protobuf messages
# This version is compatible with Debian's "protobuf-compiler" package
pip3 install "protobuf<=3.20"
+# For security-check.py and symbol-check.py
+pip3 install "lief>=0.11.4"
# Up-to-date mypy, isort and flynt packages are required python linters
pip3 install isort==5.6.4 mypy==0.780 flynt==0.78
echo "export PATH=\"$(python3 -m site --user-base)/bin:\$PATH\"" >> ~/.bashrc
# shellcheck source=/dev/null
source ~/.bashrc
# Install npm v7.x and nodejs v15.x
curl -sL https://deb.nodesource.com/setup_15.x | bash -
apt-get install -y nodejs
# Install nyc for mocha unit test reporting
npm i -g nyc
# Install Rust stable 1.67.1 and nightly from the 2023-02-17
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=1.67.1
RUST_HOME="${HOME}/.cargo/bin"
RUST_NIGHTLY_DATE=2023-02-17
"${RUST_HOME}/rustup" install nightly-${RUST_NIGHTLY_DATE}
"${RUST_HOME}/rustup" component add rustfmt --toolchain nightly-${RUST_NIGHTLY_DATE}
# Name the nightly toolchain "abc-nightly"
"${RUST_HOME}/rustup" toolchain link abc-nightly "$(${RUST_HOME}/rustc +nightly-${RUST_NIGHTLY_DATE} --print sysroot)"
# Install required compile platform targets on stable
"${RUST_HOME}/rustup" target add "i686-unknown-linux-gnu" \
"x86_64-unknown-linux-gnu" \
"aarch64-unknown-linux-gnu" \
"arm-unknown-linux-gnueabihf" \
"x86_64-apple-darwin" \
"x86_64-pc-windows-gnu"
# Install corrosion from Github
wget https://api.github.com/repos/corrosion-rs/corrosion/tarball/v0.3.0 -O corrosion.tar.gz
echo "3b9a48737264add649983df26c83f3285ce17e20d86194c7756689a0d8470267 corrosion.tar.gz" | sha256sum -c
tar xzf corrosion.tar.gz
CORROSION_SRC_FOLDER=corrosion-rs-corrosion-b764a9f
CORROSION_BUILD_FOLDER=${CORROSION_SRC_FOLDER}-build
cmake -S${CORROSION_SRC_FOLDER} -B${CORROSION_BUILD_FOLDER} -DCMAKE_BUILD_TYPE=Release
cmake --build ${CORROSION_BUILD_FOLDER} --config Release
cmake --install ${CORROSION_BUILD_FOLDER} --config Release
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Sat, Nov 23, 09:56 (1 d, 16 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
4559199
Default Alt Text
(37 KB)
Attached To
rABC Bitcoin ABC
Event Timeline
Log In to Comment