Differential D13892 Diff 40300 contrib/devtools/symbol-check.py

Changeset View

Standalone View

contrib/devtools/symbol-check.py

	#!/usr/bin/env python3			#!/usr/bin/env python3
	# Copyright (c) 2014 Wladimir J. van der Laan			# Copyright (c) 2014 Wladimir J. van der Laan
	# Distributed under the MIT software license, see the accompanying			# Distributed under the MIT software license, see the accompanying
	# file COPYING or http://www.opensource.org/licenses/mit-license.php.			# file COPYING or http://www.opensource.org/licenses/mit-license.php.
	'''			"""
	A script to check that the executables produced by gitian only contain			A script to check that the executables produced by gitian only contain
	certain symbols and are only linked against allowed libraries.			certain symbols and are only linked against allowed libraries.

	Example usage:			Example usage:

	find contrib/gitian-builder/build -type f -executable \| xargs python3 contrib/devtools/symbol-check.py			find contrib/gitian-builder/build -type f -executable \| xargs python3 contrib/devtools/symbol-check.py
	'''			"""
	import subprocess			import subprocess
	import sys			import sys
	from typing import Optional			from typing import Optional

	import lief			import lief
	import pixie			import pixie
	from utils import determine_wellknown_cmd			from utils import determine_wellknown_cmd

	# Debian 10 (Buster) EOL: 2024. https://wiki.debian.org/LTS			# Debian 10 (Buster) EOL: 2024. https://wiki.debian.org/LTS
	#			#
	# - libgcc version 8.3.0 (https://packages.debian.org/search?suite=buster&arch=any&searchon=names&keywords=libgcc1)			# - libgcc version 8.3.0 (https://packages.debian.org/search?suite=buster&arch=any&searchon=names&keywords=libgcc1)
	# - libc version 2.28 (https://packages.debian.org/search?suite=buster&arch=any&searchon=names&keywords=libc6)			# - libc version 2.28 (https://packages.debian.org/search?suite=buster&arch=any&searchon=names&keywords=libc6)
	#			#
	# Ubuntu 18.04 (Bionic) EOL: 2028. https://wiki.ubuntu.com/ReleaseTeam			# Ubuntu 18.04 (Bionic) EOL: 2028. https://wiki.ubuntu.com/ReleaseTeam
	#			#
	# - libgcc version 8.4.0 (https://packages.ubuntu.com/bionic/libgcc1)			# - libgcc version 8.4.0 (https://packages.ubuntu.com/bionic/libgcc1)
	# - libc version 2.27 (https://packages.ubuntu.com/bionic/libc6)			# - libc version 2.27 (https://packages.ubuntu.com/bionic/libc6)
	#			#
	# CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product			# CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product
	#			#
	# - libgcc version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/)			# - libgcc version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/)
	# - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/)			# - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/)
	#			#
	# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info.			# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info.
	MAX_VERSIONS = {			MAX_VERSIONS = {"GCC": (8, 3, 0), "GLIBC": (2, 27), "LIBATOMIC": (1, 0)}
	'GCC': (8, 3, 0),
	'GLIBC': (2, 27),
	'LIBATOMIC': (1, 0)
	}
	# See here for a description of _IO_stdin_used:			# See here for a description of _IO_stdin_used:
	# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109			# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109

	# Ignore symbols that are exported as part of every executable			# Ignore symbols that are exported as part of every executable
	IGNORE_EXPORTS = {			IGNORE_EXPORTS = {
	'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr',			"_edata",
				"_end",
				"__end__",
				"_init",
				"__bss_start",
				"__bss_start__",
				"_bss_end__",
				"__bss_end__",
				"_fini",
				"_IO_stdin_used",
				"stdin",
				"stdout",
				"stderr",
	# Jemalloc exported symbols			# Jemalloc exported symbols
	'__malloc_hook', 'malloc', 'calloc', 'malloc_usable_size',			"__malloc_hook",
	'__free_hook', 'free',			"malloc",
	'__realloc_hook', 'realloc',			"calloc",
	'__memalign_hook', 'memalign', 'posix_memalign', 'aligned_alloc', 'valloc',			"malloc_usable_size",
				"__free_hook",
				"free",
				"__realloc_hook",
				"realloc",
				"__memalign_hook",
				"memalign",
				"posix_memalign",
				"aligned_alloc",
				"valloc",
	# Figure out why we get these symbols exported on xenial.			# Figure out why we get these symbols exported on xenial.
	'_ZNKSt5ctypeIcE8do_widenEc', 'in6addr_any', 'optarg',			"_ZNKSt5ctypeIcE8do_widenEc",
	'_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_destroyEv'			"in6addr_any",
				"optarg",
				"_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_destroyEv",
	}			}

	# Allowed NEEDED libraries			# Allowed NEEDED libraries
	ELF_ALLOWED_LIBRARIES = {			ELF_ALLOWED_LIBRARIES = {
	# bitcoind and bitcoin-qt			# bitcoind and bitcoin-qt
	'libgcc_s.so.1', # GCC base support			"libgcc_s.so.1", # GCC base support
	'libc.so.6', # C library			"libc.so.6", # C library
	'libpthread.so.0', # threading			"libpthread.so.0", # threading
	'libanl.so.1', # DNS resolve			"libanl.so.1", # DNS resolve
	'libm.so.6', # math library			"libm.so.6", # math library
	'librt.so.1', # real-time (clock)			"librt.so.1", # real-time (clock)
	'libatomic.so.1',			"libatomic.so.1",
	'ld-linux-x86-64.so.2', # 64-bit dynamic linker			"ld-linux-x86-64.so.2", # 64-bit dynamic linker
	'ld-linux.so.2', # 32-bit dynamic linker			"ld-linux.so.2", # 32-bit dynamic linker
	'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker			"ld-linux-aarch64.so.1", # 64-bit ARM dynamic linker
	'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker			"ld-linux-armhf.so.3", # 32-bit ARM dynamic linker
	# bitcoin-qt only			# bitcoin-qt only
	'libxcb.so.1', # part of X11			"libxcb.so.1", # part of X11
	'libfontconfig.so.1', # font support			"libfontconfig.so.1", # font support
	'libfreetype.so.6', # font parsing			"libfreetype.so.6", # font parsing
	'libdl.so.2' # programming interface to dynamic linker			"libdl.so.2", # programming interface to dynamic linker
	}			}
	ARCH_MIN_GLIBC_VER = {			ARCH_MIN_GLIBC_VER = {
	pixie.EM_386: (2, 1),			pixie.EM_386: (2, 1),
	pixie.EM_X86_64: (2, 2, 5),			pixie.EM_X86_64: (2, 2, 5),
	pixie.EM_ARM: (2, 4),			pixie.EM_ARM: (2, 4),
	pixie.EM_AARCH64: (2, 17),			pixie.EM_AARCH64: (2, 17),
	}			}

	MACHO_ALLOWED_LIBRARIES = {			MACHO_ALLOWED_LIBRARIES = {
	# bitcoind and bitcoin-qt			# bitcoind and bitcoin-qt
	'libc++.1.dylib', # C++ Standard Library			"libc++.1.dylib", # C++ Standard Library
	'libSystem.B.dylib', # libc, libm, libpthread, libinfo			"libSystem.B.dylib", # libc, libm, libpthread, libinfo
	# bitcoin-qt only			# bitcoin-qt only
	'AppKit', # user interface			"AppKit", # user interface
	'ApplicationServices', # common application tasks.			"ApplicationServices", # common application tasks.
	'Carbon', # deprecated c back-compat API			"Carbon", # deprecated c back-compat API
	'CFNetwork', # network services and changes in network configurations			"CFNetwork", # network services and changes in network configurations
	'CoreFoundation', # low level func, data types			"CoreFoundation", # low level func, data types
	'CoreGraphics', # 2D rendering			"CoreGraphics", # 2D rendering
	'CoreServices', # operating system services			"CoreServices", # operating system services
	'CoreText', # interface for laying out text and handling fonts.			"CoreText", # interface for laying out text and handling fonts.
	'Foundation', # base layer functionality for apps/frameworks			"Foundation", # base layer functionality for apps/frameworks
	'ImageIO', # read and write image file formats.			"ImageIO", # read and write image file formats.
	'IOKit', # user-space access to hardware devices and drivers.			"IOKit", # user-space access to hardware devices and drivers.
	'libobjc.A.dylib', # Objective-C runtime library			"libobjc.A.dylib", # Objective-C runtime library
	'Security', # access control and authentication			"Security", # access control and authentication
	'SystemConfiguration', # access network configuration settings			"SystemConfiguration", # access network configuration settings
	}			}

	PE_ALLOWED_LIBRARIES = {			PE_ALLOWED_LIBRARIES = {
	'ADVAPI32.dll', # security & registry			"ADVAPI32.dll", # security & registry
	'IPHLPAPI.DLL', # IP helper API			"IPHLPAPI.DLL", # IP helper API
	'KERNEL32.dll', # win32 base APIs			"KERNEL32.dll", # win32 base APIs
	'msvcrt.dll', # C standard library for MSVC			"msvcrt.dll", # C standard library for MSVC
	'SHELL32.dll', # shell API			"SHELL32.dll", # shell API
	'USER32.dll', # user interface			"USER32.dll", # user interface
	'WS2_32.dll', # sockets			"WS2_32.dll", # sockets
	# bitcoin-qt only			# bitcoin-qt only
	'dwmapi.dll', # desktop window manager			"dwmapi.dll", # desktop window manager
	'CRYPT32.dll', # openssl			"CRYPT32.dll", # openssl
	'GDI32.dll', # graphics device interface			"GDI32.dll", # graphics device interface
	'IMM32.dll', # input method editor			"IMM32.dll", # input method editor
	'ole32.dll', # component object model			"ole32.dll", # component object model
	'OLEAUT32.dll', # OLE Automation API			"OLEAUT32.dll", # OLE Automation API
	'SHLWAPI.dll', # light weight shell API			"SHLWAPI.dll", # light weight shell API
	'UxTheme.dll',			"UxTheme.dll",
	'VERSION.dll', # version checking			"VERSION.dll", # version checking
	'WINMM.dll', # WinMM audio API			"WINMM.dll", # WinMM audio API
	}			}


	class CPPFilt(object):			class CPPFilt(object):
	'''			"""
	Demangle C++ symbol names.			Demangle C++ symbol names.

	Use a pipe to the 'c++filt' command.			Use a pipe to the 'c++filt' command.
	'''			"""

	def __init__(self):			def __init__(self):
	self.proc = subprocess.Popen(			self.proc = subprocess.Popen(
	determine_wellknown_cmd('CPPFILT', 'c++filt'),			determine_wellknown_cmd("CPPFILT", "c++filt"),
	stdin=subprocess.PIPE,			stdin=subprocess.PIPE,
	stdout=subprocess.PIPE,			stdout=subprocess.PIPE,
	universal_newlines=True)			universal_newlines=True,
				)

	def __call__(self, mangled):			def __call__(self, mangled):
	self.proc.stdin.write(mangled + '\n')			self.proc.stdin.write(mangled + "\n")
	self.proc.stdin.flush()			self.proc.stdin.flush()
	return self.proc.stdout.readline().rstrip()			return self.proc.stdout.readline().rstrip()

	def close(self):			def close(self):
	self.proc.stdin.close()			self.proc.stdin.close()
	self.proc.stdout.close()			self.proc.stdout.close()
	self.proc.wait()			self.proc.wait()


	def check_version(max_versions, version, arch) -> bool:			def check_version(max_versions, version, arch) -> bool:
	if '_' in version:			if "_" in version:
	(lib, _, ver) = version.rpartition('_')			(lib, _, ver) = version.rpartition("_")
	else:			else:
	lib = version			lib = version
	ver = '0'			ver = "0"
	ver = tuple([int(x) for x in ver.split('.')])			ver = tuple([int(x) for x in ver.split(".")])
	if lib not in max_versions:			if lib not in max_versions:
	return False			return False
	return ver <= max_versions[lib] or lib == 'GLIBC' and ver <= ARCH_MIN_GLIBC_VER[arch]			return (
				ver <= max_versions[lib] or lib == "GLIBC" and ver <= ARCH_MIN_GLIBC_VER[arch]
				)


	def check_imported_symbols(filename) -> bool:			def check_imported_symbols(filename) -> bool:
	elf = pixie.load(filename)			elf = pixie.load(filename)
	cppfilt = CPPFilt()			cppfilt = CPPFilt()
	ok = True			ok = True

	for symbol in elf.dyn_symbols:			for symbol in elf.dyn_symbols:
	if not symbol.is_import:			if not symbol.is_import:
	continue			continue
	sym = symbol.name.decode()			sym = symbol.name.decode()
	version = symbol.version.decode() if symbol.version is not None else None			version = symbol.version.decode() if symbol.version is not None else None
	if version and not check_version(MAX_VERSIONS, version, elf.hdr.e_machine):			if version and not check_version(MAX_VERSIONS, version, elf.hdr.e_machine):
	print(f'{filename}: symbol {cppfilt(sym)} from unsupported version '			print(
	f'{version}')			f"{filename}: symbol {cppfilt(sym)} from unsupported version {version}"
				)
	ok = False			ok = False
	return ok			return ok


	def check_exported_symbols(filename) -> bool:			def check_exported_symbols(filename) -> bool:
	elf = pixie.load(filename)			elf = pixie.load(filename)
	cppfilt = CPPFilt()			cppfilt = CPPFilt()
	ok = True			ok = True
	for symbol in elf.dyn_symbols:			for symbol in elf.dyn_symbols:
	if not symbol.is_export:			if not symbol.is_export:
	continue			continue
	sym = symbol.name.decode()			sym = symbol.name.decode()
	if sym in IGNORE_EXPORTS:			if sym in IGNORE_EXPORTS:
	continue			continue
	print(f'{filename}: export of symbol {cppfilt(sym)} not allowed')			print(f"{filename}: export of symbol {cppfilt(sym)} not allowed")
	ok = False			ok = False
	return ok			return ok


	def check_ELF_libraries(filename) -> bool:			def check_ELF_libraries(filename) -> bool:
	ok = True			ok = True
	elf = pixie.load(filename)			elf = pixie.load(filename)
	for library_name in elf.query_dyn_tags(pixie.DT_NEEDED):			for library_name in elf.query_dyn_tags(pixie.DT_NEEDED):
	assert isinstance(library_name, bytes)			assert isinstance(library_name, bytes)
	if library_name.decode() not in ELF_ALLOWED_LIBRARIES:			if library_name.decode() not in ELF_ALLOWED_LIBRARIES:
	print(f'{filename}: NEEDED library {library_name.decode()} is not allowed')			print(f"{filename}: NEEDED library {library_name.decode()} is not allowed")
	ok = False			ok = False
	return ok			return ok


	def check_MACHO_libraries(filename) -> bool:			def check_MACHO_libraries(filename) -> bool:
	ok: bool = True			ok: bool = True
	binary = lief.parse(filename)			binary = lief.parse(filename)
	for dylib in binary.libraries:			for dylib in binary.libraries:
	split = dylib.name.split('/')			split = dylib.name.split("/")
	if split[-1] not in MACHO_ALLOWED_LIBRARIES:			if split[-1] not in MACHO_ALLOWED_LIBRARIES:
	print(f'{split[-1]} is not in ALLOWED_LIBRARIES!')			print(f"{split[-1]} is not in ALLOWED_LIBRARIES!")
	ok = False			ok = False
	return ok			return ok


	def check_MACHO_min_os(filename) -> bool:			def check_MACHO_min_os(filename) -> bool:
	binary = lief.parse(filename)			binary = lief.parse(filename)
	return binary.build_version.minos == [10, 15, 0]			return binary.build_version.minos == [10, 15, 0]


	def check_MACHO_sdk(filename) -> bool:			def check_MACHO_sdk(filename) -> bool:
	binary = lief.parse(filename)			binary = lief.parse(filename)
	return binary.build_version.sdk == [10, 15, 6]			return binary.build_version.sdk == [10, 15, 6]


	def check_PE_libraries(filename) -> bool:			def check_PE_libraries(filename) -> bool:
	ok: bool = True			ok: bool = True
	binary = lief.parse(filename)			binary = lief.parse(filename)
	for dylib in binary.libraries:			for dylib in binary.libraries:
	if dylib not in PE_ALLOWED_LIBRARIES:			if dylib not in PE_ALLOWED_LIBRARIES:
	print(f'{dylib} is not in ALLOWED_LIBRARIES!')			print(f"{dylib} is not in ALLOWED_LIBRARIES!")
	ok = False			ok = False
	return ok			return ok


	def check_PE_subsystem_version(filename) -> bool:			def check_PE_subsystem_version(filename) -> bool:
	binary = lief.parse(filename)			binary = lief.parse(filename)
	major: int = binary.optional_header.major_subsystem_version			major: int = binary.optional_header.major_subsystem_version
	minor: int = binary.optional_header.minor_subsystem_version			minor: int = binary.optional_header.minor_subsystem_version
	return major == 6 and minor == 1			return major == 6 and minor == 1


	CHECKS = {			CHECKS = {
	'ELF': [			"ELF": [
	('IMPORTED_SYMBOLS', check_imported_symbols),			("IMPORTED_SYMBOLS", check_imported_symbols),
	('EXPORTED_SYMBOLS', check_exported_symbols),			("EXPORTED_SYMBOLS", check_exported_symbols),
	('LIBRARY_DEPENDENCIES', check_ELF_libraries)			("LIBRARY_DEPENDENCIES", check_ELF_libraries),
				],
				"MACHO": [
				("DYNAMIC_LIBRARIES", check_MACHO_libraries),
				("MIN_OS", check_MACHO_min_os),
				("SDK", check_MACHO_sdk),
	],			],
	'MACHO': [			"PE": [
	('DYNAMIC_LIBRARIES', check_MACHO_libraries),			("DYNAMIC_LIBRARIES", check_PE_libraries),
	('MIN_OS', check_MACHO_min_os),			("SUBSYSTEM_VERSION", check_PE_subsystem_version),
	('SDK', check_MACHO_sdk),
	],			],
	'PE': [
	('DYNAMIC_LIBRARIES', check_PE_libraries),
	('SUBSYSTEM_VERSION', check_PE_subsystem_version),
	]
	}			}


	def identify_executable(filename) -> Optional[str]:			def identify_executable(filename) -> Optional[str]:
	with open(filename, 'rb') as f:			with open(filename, "rb") as f:
	magic = f.read(4)			magic = f.read(4)
	if magic.startswith(b'MZ'):			if magic.startswith(b"MZ"):
	return 'PE'			return "PE"
	elif magic.startswith(b'\x7fELF'):			elif magic.startswith(b"\x7fELF"):
	return 'ELF'			return "ELF"
	elif magic.startswith(b'\xcf\xfa'):			elif magic.startswith(b"\xcf\xfa"):
	return 'MACHO'			return "MACHO"
	return None			return None


	if __name__ == '__main__':			if __name__ == "__main__":
	retval = 0			retval = 0
	for filename in sys.argv[1:]:			for filename in sys.argv[1:]:
	try:			try:
	etype = identify_executable(filename)			etype = identify_executable(filename)
	if etype is None:			if etype is None:
	print(f'{filename}: unknown format')			print(f"{filename}: unknown format")
	retval = 1			retval = 1
	continue			continue

	failed = []			failed = []
	for (name, func) in CHECKS[etype]:			for name, func in CHECKS[etype]:
	if not func(filename):			if not func(filename):
	failed.append(name)			failed.append(name)
	if failed:			if failed:
	print(f'{filename}: failed {" ".join(failed)}')			print(f'{filename}: failed {" ".join(failed)}')
	retval = 1			retval = 1
	except IOError:			except IOError:
	print(f'{filename}: cannot open')			print(f"{filename}: cannot open")
	retval = 1			retval = 1
	sys.exit(retval)			sys.exit(retval)