diff --git a/contrib/apple-sdk-tools/extract_xcode.py b/contrib/apple-sdk-tools/extract_xcode.py index 6156b8c52..1dda41e05 100755 --- a/contrib/apple-sdk-tools/extract_xcode.py +++ b/contrib/apple-sdk-tools/extract_xcode.py @@ -1,130 +1,130 @@ #!/usr/bin/env python3 # Copyright (c) 2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """XCode extractor """ import argparse import sys import struct import zlib import xml.etree.ElementTree as ET import lzma XAR_MAGIC = b'\x78\x61\x72\x21' PBZX_MAGIC = b'\x70\x62\x7a\x78' LZMA_MAGIC = b'\xfd\x37\x7a\x58\x5a\x00' class io_wrapper(object): """Helper for stdin/stdout binary weirdness""" def __init__(self, filename, mode): self.filename = filename self.mode = mode def __enter__(self): if self.filename == '-': if self.mode is None or self.mode == '' or 'r' in self.mode: self.fh = sys.stdin else: self.fh = sys.stdout else: self.fh = open(self.filename, self.mode) return self def __exit__(self, exc_type, exc_val, exc_tb): if self.filename != '-': self.fh.close() - def write(self, bytes): + def write(self, b): if self.filename != '-': - return self.fh.write(bytes) - return self.fh.buffer.write(bytes) + return self.fh.write(b) + return self.fh.buffer.write(b) def read(self, size): if self.filename != '-': return self.fh.read(size) return self.fh.buffer.read(size) def seek(self, size): return self.fh.seek(size) def run(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-f", '--file', nargs='?', default="-") parser.add_argument('outfile', nargs='?', default="-") args = parser.parse_args() with io_wrapper(args.file, "rb") as infile, io_wrapper(args.outfile, "wb") as outfile: start_offset = 0 magic = infile.read(4) if magic != XAR_MAGIC: print("bad xar magic", file=sys.stderr) sys.exit(1) - bytes = infile.read(24) - header_size, xar_version, toc_compressed, toc_uncompressed, checksum_type = struct.unpack('>HHQQI', bytes) + in_bytes = infile.read(24) + header_size, xar_version, toc_compressed, toc_uncompressed, checksum_type = struct.unpack('>HHQQI', in_bytes) start_offset += header_size start_offset += toc_compressed - bytes = infile.read(toc_compressed) + in_bytes = infile.read(toc_compressed) - xml_toc = zlib.decompress(bytes).decode("utf-8") + xml_toc = zlib.decompress(in_bytes).decode("utf-8") root = ET.fromstring(xml_toc) content_offset = 0 content_length = 0 content_encoding = "" toc = root.find("toc") for elem in toc.findall("file"): name = elem.find("name").text if name == "Content": data = elem.find("data") content_offset = int(data.find("offset").text) content_length = int(data.find("length").text) content_encoding = data.find("encoding").get("style") content_uncompressed_length = int(data.find("size").text) found_content = True break if (content_length == 0): print("No \"Content\" file to extract.", file=sys.stderr) sys.exit(1) infile.seek(content_offset + start_offset) content_read_size = 0 magic = infile.read(4) content_read_size += 4 if magic != PBZX_MAGIC: print("bad pbzx magic", file=sys.stderr) sys.exit(2) - bytes = infile.read(8) + in_bytes = infile.read(8) content_read_size += 8 - flags, = struct.unpack('>Q', bytes) - bytes = infile.read(16) + flags, = struct.unpack('>Q', in_bytes) + in_bytes = infile.read(16) content_read_size += 16 while (flags & 1 << 24): - flags, size = struct.unpack('>QQ', bytes) - bytes = infile.read(size) + flags, size = struct.unpack('>QQ', in_bytes) + in_bytes = infile.read(size) content_read_size += size compressed = size != 1 << 24 if compressed: - if bytes[0:6] != LZMA_MAGIC: + if in_bytes[0:6] != LZMA_MAGIC: print("bad lzma magic: ", file=sys.stderr) sys.exit(3) - outfile.write(lzma.decompress(bytes)) + outfile.write(lzma.decompress(in_bytes)) else: - outfile.write(bytes) + outfile.write(in_bytes) if content_read_size == content_length: break - - bytes = infile.read(16) + + in_bytes = infile.read(16) content_read_size += 16 if __name__ == '__main__': run() diff --git a/contrib/macdeploy/macdeployqtplus.py b/contrib/macdeploy/macdeployqtplus.py index b1f7a6b7b..f2cbfa551 100755 --- a/contrib/macdeploy/macdeployqtplus.py +++ b/contrib/macdeploy/macdeployqtplus.py @@ -1,1124 +1,1124 @@ #!/usr/bin/env python3 # # Copyright (C) 2011 Patrick "p2k" Schneider # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # import os import os.path import re import shutil import stat import subprocess import sys import time from argparse import ArgumentParser from string import Template from typing import List, Optional # This is ported from the original macdeployqt with modifications class FrameworkInfo(object): def __init__(self): self.frameworkDirectory = "" self.frameworkName = "" self.frameworkPath = "" self.binaryDirectory = "" self.binaryName = "" self.binaryPath = "" self.version = "" self.installName = "" self.deployedInstallName = "" self.sourceFilePath = "" self.destinationDirectory = "" self.sourceResourcesDirectory = "" self.sourceVersionContentsDirectory = "" self.sourceContentsDirectory = "" self.destinationResourcesDirectory = "" self.destinationVersionContentsDirectory = "" def __eq__(self, other): if self.__class__ == other.__class__: return self.__dict__ == other.__dict__ else: return False def __str__(self): return f""" Framework name: {self.frameworkName} Framework directory: {self.frameworkDirectory} Framework path: {self.frameworkPath} Binary name: {self.binaryName} Binary directory: {self.binaryDirectory} Binary path: {self.binaryPath} Version: {self.version} Install name: {self.installName} Deployed install name: {self.deployedInstallName} Source file Path: {self.sourceFilePath} Deployed Directory (relative to bundle): {self.destinationDirectory} """ def isDylib(self): return self.frameworkName.endswith(".dylib") def isQtFramework(self): if self.isDylib(): return self.frameworkName.startswith("libQt") else: return self.frameworkName.startswith("Qt") reOLine = re.compile( r'^(.+) \(compatibility version [0-9.]+, current version [0-9.]+\)$') bundleFrameworkDirectory = "Contents/Frameworks" bundleBinaryDirectory = "Contents/MacOS" @classmethod def fromOtoolLibraryLine(cls, line: str) -> Optional['FrameworkInfo']: # Note: line must be trimmed if line == "": return None # Don't deploy system libraries (exception for libQtuitools and # libQtlucene). if line.startswith("/System/Library/") or line.startswith( "@executable_path") or (line.startswith("/usr/lib/") and "libQt" not in line): return None m = cls.reOLine.match(line) if m is None: raise RuntimeError("otool line could not be parsed: " + line) path = m.group(1) info = cls() info.sourceFilePath = path info.installName = path if path.endswith(".dylib"): dirname, filename = os.path.split(path) info.frameworkName = filename info.frameworkDirectory = dirname info.frameworkPath = path info.binaryDirectory = dirname info.binaryName = filename info.binaryPath = path info.version = "-" info.installName = path info.deployedInstallName = "@executable_path/../Frameworks/" + info.binaryName info.sourceFilePath = path info.destinationDirectory = cls.bundleFrameworkDirectory else: parts = path.split("/") i = 0 # Search for the .framework directory for part in parts: if part.endswith(".framework"): break i += 1 if i == len(parts): raise RuntimeError( "Could not find .framework or .dylib in otool line: " + line) info.frameworkName = parts[i] info.frameworkDirectory = "/".join(parts[:i]) info.frameworkPath = os.path.join( info.frameworkDirectory, info.frameworkName) info.binaryName = parts[i + 3] info.binaryDirectory = "/".join(parts[i + 1:i + 3]) info.binaryPath = os.path.join( info.binaryDirectory, info.binaryName) info.version = parts[i + 2] info.deployedInstallName = "@executable_path/../Frameworks/" + \ os.path.join(info.frameworkName, info.binaryPath) info.destinationDirectory = os.path.join( cls.bundleFrameworkDirectory, info.frameworkName, info.binaryDirectory) info.sourceResourcesDirectory = os.path.join( info.frameworkPath, "Resources") info.sourceContentsDirectory = os.path.join( info.frameworkPath, "Contents") info.sourceVersionContentsDirectory = os.path.join( info.frameworkPath, "Versions", info.version, "Contents") info.destinationResourcesDirectory = os.path.join( cls.bundleFrameworkDirectory, info.frameworkName, "Resources") info.destinationVersionContentsDirectory = os.path.join( cls.bundleFrameworkDirectory, info.frameworkName, "Versions", info.version, "Contents") return info class ApplicationBundleInfo(object): def __init__(self, path: str): self.path = path appName = "BitcoinABC-Qt" self.binaryPath = os.path.join(path, "Contents", "MacOS", appName) if not os.path.exists(self.binaryPath): raise RuntimeError("Could not find bundle binary for " + path) self.resourcesPath = os.path.join(path, "Contents", "Resources") self.pluginPath = os.path.join(path, "Contents", "PlugIns") class DeploymentInfo(object): def __init__(self): self.qtPath = None self.pluginPath = None self.deployedFrameworks = [] def detectQtPath(self, frameworkDirectory: str): parentDir = os.path.dirname(frameworkDirectory) if os.path.exists(os.path.join(parentDir, "translations")): # Classic layout, e.g. "/usr/local/Trolltech/Qt-4.x.x" self.qtPath = parentDir else: self.qtPath = os.getenv("QTDIR", None) if self.qtPath is not None: pluginPath = os.path.join(self.qtPath, "plugins") if os.path.exists(pluginPath): self.pluginPath = pluginPath def usesFramework(self, name: str) -> bool: nameDot = f"{name}." libNameDot = f"lib{name}." for framework in self.deployedFrameworks: if framework.endswith(".framework"): if framework.startswith(nameDot): return True elif framework.endswith(".dylib"): if framework.startswith(libNameDot): return True return False def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: if verbose >= 3: print("Inspecting with otool: " + binaryPath) otoolbin = os.getenv("OTOOL", "otool") otool = subprocess.Popen([otoolbin, "-L", binaryPath], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) o_stdout, o_stderr = otool.communicate() if otool.returncode != 0: if verbose >= 1: sys.stderr.write(o_stderr) sys.stderr.flush() raise RuntimeError( f"otool failed with return code {otool.returncode}") otoolLines = o_stdout.split("\n") otoolLines.pop(0) # First line is the inspected binary if ".framework" in binaryPath or binaryPath.endswith(".dylib"): # Frameworks and dylibs list themselves as a dependency. otoolLines.pop(0) libraries = [] for line in otoolLines: line = line.replace("@loader_path", os.path.dirname(binaryPath)) info = FrameworkInfo.fromOtoolLibraryLine(line.strip()) if info is not None: if verbose >= 3: print("Found framework:") print(info) libraries.append(info) return libraries def runInstallNameTool(action: str, *args): installnametoolbin = os.getenv("INSTALLNAMETOOL", "install_name_tool") subprocess.check_call([installnametoolbin, "-" + action] + list(args)) def changeInstallName(oldName: str, newName: str, binaryPath: str, verbose: int): if verbose >= 3: print("Using install_name_tool:") print(" in", binaryPath) print(" change reference", oldName) print(" to", newName) runInstallNameTool("change", oldName, newName, binaryPath) -def changeIdentification(id: str, binaryPath: str, verbose: int): +def changeIdentification(id_name: str, binaryPath: str, verbose: int): if verbose >= 3: print("Using install_name_tool:") print(" change identification in", binaryPath) - print(" to", id) - runInstallNameTool("id", id, binaryPath) + print(" to", id_name) + runInstallNameTool("id", id_name, binaryPath) def runStrip(binaryPath: str, verbose: int): stripbin = os.getenv("STRIP", "strip") if verbose >= 3: print("Using strip:") print(" stripped", binaryPath) subprocess.check_call([stripbin, "-x", binaryPath]) def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional[str]: if framework.sourceFilePath.startswith("Qt"): # standard place for Nokia Qt installer's frameworks fromPath = "/Library/Frameworks/" + framework.sourceFilePath else: fromPath = framework.sourceFilePath toDir = os.path.join(path, framework.destinationDirectory) toPath = os.path.join(toDir, framework.binaryName) if not os.path.exists(fromPath): raise RuntimeError("No file at " + fromPath) if os.path.exists(toPath): return None # Already there if not os.path.exists(toDir): os.makedirs(toDir) shutil.copy2(fromPath, toPath) if verbose >= 3: print("Copied:", fromPath) print(" to:", toPath) permissions = os.stat(toPath) if not permissions.st_mode & stat.S_IWRITE: os.chmod(toPath, permissions.st_mode | stat.S_IWRITE) if not framework.isDylib(): # Copy resources for real frameworks linkfrom = os.path.join( path, "Contents", "Frameworks", framework.frameworkName, "Versions", "Current") linkto = framework.version if not os.path.exists(linkfrom): os.symlink(linkto, linkfrom) if verbose >= 2: print("Linked:", linkfrom, "->", linkto) fromResourcesDir = framework.sourceResourcesDirectory if os.path.exists(fromResourcesDir): toResourcesDir = os.path.join( path, framework.destinationResourcesDirectory) shutil.copytree(fromResourcesDir, toResourcesDir, symlinks=True) if verbose >= 3: print("Copied resources:", fromResourcesDir) print(" to:", toResourcesDir) fromContentsDir = framework.sourceVersionContentsDirectory if not os.path.exists(fromContentsDir): fromContentsDir = framework.sourceContentsDirectory if os.path.exists(fromContentsDir): toContentsDir = os.path.join( path, framework.destinationVersionContentsDirectory) shutil.copytree(fromContentsDir, toContentsDir, symlinks=True) if verbose >= 3: print("Copied Contents:", fromContentsDir) print(" to:", toContentsDir) # Copy qt_menu.nib (applies to non-framework layout) elif framework.frameworkName.startswith("libQtGui"): qtMenuNibSourcePath = os.path.join( framework.frameworkDirectory, "Resources", "qt_menu.nib") qtMenuNibDestinationPath = os.path.join( path, "Contents", "Resources", "qt_menu.nib") if os.path.exists(qtMenuNibSourcePath) and not os.path.exists( qtMenuNibDestinationPath): shutil.copytree( qtMenuNibSourcePath, qtMenuNibDestinationPath, symlinks=True) if verbose >= 3: print("Copied for libQtGui:", qtMenuNibSourcePath) print(" to:", qtMenuNibDestinationPath) return toPath def deployFrameworks(frameworks: List[FrameworkInfo], bundlePath: str, binaryPath: str, strip: bool, verbose: int, deploymentInfo: Optional[DeploymentInfo] = None) -> DeploymentInfo: if deploymentInfo is None: deploymentInfo = DeploymentInfo() while len(frameworks) > 0: framework = frameworks.pop(0) deploymentInfo.deployedFrameworks.append(framework.frameworkName) if verbose >= 2: print("Processing", framework.frameworkName, "...") # Get the Qt path from one of the Qt frameworks if deploymentInfo.qtPath is None and framework.isQtFramework(): deploymentInfo.detectQtPath(framework.frameworkDirectory) if framework.installName.startswith( "@executable_path") or framework.installName.startswith(bundlePath): if verbose >= 2: print(framework.frameworkName, "already deployed, skipping.") continue # install_name_tool the new id into the binary changeInstallName( framework.installName, framework.deployedInstallName, binaryPath, verbose) # Copy framework to app bundle. deployedBinaryPath = copyFramework(framework, bundlePath, verbose) # Skip the rest if already was deployed. if deployedBinaryPath is None: continue if strip: runStrip(deployedBinaryPath, verbose) # install_name_tool it a new id. changeIdentification( framework.deployedInstallName, deployedBinaryPath, verbose) # Check for framework dependencies dependencies = getFrameworks(deployedBinaryPath, verbose) for dependency in dependencies: changeInstallName( dependency.installName, dependency.deployedInstallName, deployedBinaryPath, verbose) # Deploy framework if necessary. if dependency.frameworkName not in deploymentInfo.deployedFrameworks and dependency not in frameworks: frameworks.append(dependency) return deploymentInfo def deployFrameworksForAppBundle( applicationBundle: ApplicationBundleInfo, strip: bool, verbose: int) -> DeploymentInfo: frameworks = getFrameworks(applicationBundle.binaryPath, verbose) if len(frameworks) == 0 and verbose >= 1: print( "Warning: Could not find any external frameworks to deploy in {}.".format( applicationBundle.path)) return DeploymentInfo() else: return deployFrameworks( frameworks, applicationBundle.path, applicationBundle.binaryPath, strip, verbose) def deployPlugins(appBundleInfo: ApplicationBundleInfo, deploymentInfo: DeploymentInfo, strip: bool, verbose: int): # Lookup available plugins, exclude unneeded plugins = [] if deploymentInfo.pluginPath is None: return for dirpath, dirnames, filenames in os.walk(deploymentInfo.pluginPath): pluginDirectory = os.path.relpath(dirpath, deploymentInfo.pluginPath) if pluginDirectory == "designer": # Skip designer plugins continue elif pluginDirectory == "printsupport": # Skip printsupport plugins continue elif pluginDirectory == "imageformats": # Skip imageformats plugins continue elif pluginDirectory == "sqldrivers": # Deploy the sql plugins only if QtSql is in use if not deploymentInfo.usesFramework("QtSql"): continue elif pluginDirectory == "script": # Deploy the script plugins only if QtScript is in use if not deploymentInfo.usesFramework("QtScript"): continue elif pluginDirectory == "qmltooling" or pluginDirectory == "qml1tooling": # Deploy the qml plugins only if QtDeclarative is in use if not deploymentInfo.usesFramework("QtDeclarative"): continue elif pluginDirectory == "bearer": # Deploy the bearer plugins only if QtNetwork is in use if not deploymentInfo.usesFramework("QtNetwork"): continue elif pluginDirectory == "position": # Deploy the position plugins only if QtPositioning is in use if not deploymentInfo.usesFramework("QtPositioning"): continue elif pluginDirectory == "sensors" or pluginDirectory == "sensorgestures": # Deploy the sensor plugins only if QtSensors is in use if not deploymentInfo.usesFramework("QtSensors"): continue elif pluginDirectory == "audio" or pluginDirectory == "playlistformats": # Deploy the audio plugins only if QtMultimedia is in use if not deploymentInfo.usesFramework("QtMultimedia"): continue elif pluginDirectory == "mediaservice": # Deploy the mediaservice plugins only if QtMultimediaWidgets is in # use if not deploymentInfo.usesFramework("QtMultimediaWidgets"): continue elif pluginDirectory == "canbus": # Deploy the canbus plugins only if QtSerialBus is in use if not deploymentInfo.usesFramework("QtSerialBus"): continue elif pluginDirectory == "webview": # Deploy the webview plugins only if QtWebView is in use if not deploymentInfo.usesFramework("QtWebView"): continue elif pluginDirectory == "gamepads": # Deploy the webview plugins only if QtGamepad is in use if not deploymentInfo.usesFramework("QtGamepad"): continue elif pluginDirectory == "geoservices": # Deploy the webview plugins only if QtLocation is in use if not deploymentInfo.usesFramework("QtLocation"): continue elif pluginDirectory == "texttospeech": # Deploy the texttospeech plugins only if QtTextToSpeech is in use if not deploymentInfo.usesFramework("QtTextToSpeech"): continue elif pluginDirectory == "virtualkeyboard": # Deploy the virtualkeyboard plugins only if QtVirtualKeyboard is # in use if not deploymentInfo.usesFramework("QtVirtualKeyboard"): continue elif pluginDirectory == "sceneparsers": # Deploy the virtualkeyboard plugins only if Qt3DCore is in use if not deploymentInfo.usesFramework("Qt3DCore"): continue elif pluginDirectory == "renderplugins": # Deploy the renderplugins plugins only if Qt3DCore is in use if not deploymentInfo.usesFramework("Qt3DCore"): continue elif pluginDirectory == "geometryloaders": # Deploy the geometryloaders plugins only if Qt3DCore is in use if not deploymentInfo.usesFramework("Qt3DCore"): continue for pluginName in filenames: pluginPath = os.path.join(pluginDirectory, pluginName) if pluginName.endswith("_debug.dylib"): # Skip debug plugins continue elif pluginPath == "imageformats/libqsvg.dylib" or pluginPath == "iconengines/libqsvgicon.dylib": # Deploy the svg plugins only if QtSvg is in use if not deploymentInfo.usesFramework("QtSvg"): continue elif pluginPath == "accessible/libqtaccessiblecompatwidgets.dylib": # Deploy accessibility for Qt3Support only if the Qt3Support is # in use if not deploymentInfo.usesFramework("Qt3Support"): continue elif pluginPath == "graphicssystems/libqglgraphicssystem.dylib": # Deploy the opengl graphicssystem plugin only if QtOpenGL is # in use if not deploymentInfo.usesFramework("QtOpenGL"): continue elif pluginPath == "accessible/libqtaccessiblequick.dylib": # Deploy the accessible qtquick plugin only if QtQuick is in # use if not deploymentInfo.usesFramework("QtQuick"): continue elif pluginPath == "platforminputcontexts/libqtvirtualkeyboardplugin.dylib": # Deploy the virtualkeyboardplugin plugin only if # QtVirtualKeyboard is in use if not deploymentInfo.usesFramework("QtVirtualKeyboard"): continue plugins.append((pluginDirectory, pluginName)) for pluginDirectory, pluginName in plugins: if verbose >= 2: print( "Processing plugin", os.path.join( pluginDirectory, pluginName), "...") sourcePath = os.path.join( deploymentInfo.pluginPath, pluginDirectory, pluginName) destinationDirectory = os.path.join( appBundleInfo.pluginPath, pluginDirectory) if not os.path.exists(destinationDirectory): os.makedirs(destinationDirectory) destinationPath = os.path.join(destinationDirectory, pluginName) shutil.copy2(sourcePath, destinationPath) if verbose >= 3: print("Copied:", sourcePath) print(" to:", destinationPath) if strip: runStrip(destinationPath, verbose) dependencies = getFrameworks(destinationPath, verbose) for dependency in dependencies: changeInstallName( dependency.installName, dependency.deployedInstallName, destinationPath, verbose) # Deploy framework if necessary. if dependency.frameworkName not in deploymentInfo.deployedFrameworks: deployFrameworks( [dependency], appBundleInfo.path, destinationPath, strip, verbose, deploymentInfo) qt_conf = """[Paths] Translations=Resources Plugins=PlugIns """ ap = ArgumentParser(description="""Improved version of macdeployqt. Outputs a ready-to-deploy app in a folder "dist" and optionally wraps it in a .dmg file. Note, that the "dist" folder will be deleted before deploying on each run. Optionally, Qt translation files (.qm) and additional resources can be added to the bundle. Also optionally signs the .app bundle; set the CODESIGNARGS environment variable to pass arguments to the codesign tool. E.g. CODESIGNARGS='--sign "Developer ID Application: ..." --keychain /encrypted/foo.keychain'""") ap.add_argument("app_bundle", nargs=1, metavar="app-bundle", help="application bundle to be deployed") ap.add_argument( "-verbose", type=int, nargs=1, default=[1], metavar="<0-3>", help="0 = no output, 1 = error/warning (default), 2 = normal, 3 = debug") ap.add_argument( "-no-plugins", dest="plugins", action="store_false", default=True, help="skip plugin deployment") ap.add_argument( "-no-strip", dest="strip", action="store_false", default=True, help="don't run 'strip' on the binaries") ap.add_argument( "-sign", dest="sign", action="store_true", default=False, help="sign .app bundle with codesign tool") ap.add_argument( "-dmg", nargs="?", const="", metavar="basename", help="create a .dmg disk image; if basename is not specified, a camel-cased version of the app name is used") ap.add_argument( "-fancy", nargs=1, metavar="plist", default=[], help="make a fancy looking disk image using the given plist file with instructions; requires -dmg to work") ap.add_argument( "-add-qt-tr", nargs=1, metavar="languages", default=[], help="add Qt translation files to the bundle's resources; the language list must be separated with commas, not with whitespace") ap.add_argument( "-translations-dir", nargs=1, metavar="path", default=None, help="Path to Qt's translation files") ap.add_argument( "-add-resources", nargs="+", metavar="path", default=[], help="list of additional files or folders to be copied into the bundle's resources; must be the last argument") ap.add_argument( "-volname", nargs=1, metavar="volname", default=[], help="custom volume name for dmg") config = ap.parse_args() verbose = config.verbose[0] # ------------------------------------------------ app_bundle = config.app_bundle[0] if not os.path.exists(app_bundle): if verbose >= 1: sys.stderr.write( f"Error: Could not find app bundle \"{app_bundle}\"\n") sys.exit(1) app_bundle_name = os.path.splitext(os.path.basename(app_bundle))[0] # ------------------------------------------------ translations_dir = None if config.translations_dir and config.translations_dir[0]: if os.path.exists(config.translations_dir[0]): translations_dir = config.translations_dir[0] else: if verbose >= 1: sys.stderr.write( f"Error: Could not find translation dir \"{translations_dir}\"\n") sys.exit(1) # ------------------------------------------------ for p in config.add_resources: if verbose >= 3: print(f"Checking for \"{p}\"...") if not os.path.exists(p): if verbose >= 1: sys.stderr.write( f"Error: Could not find additional resource file \"{p}\"\n") sys.exit(1) # ------------------------------------------------ if len(config.fancy) == 1: if verbose >= 3: print("Fancy: Importing plistlib...") try: import plistlib except ImportError: if verbose >= 1: sys.stderr.write( "Error: Could not import plistlib which is required for fancy disk images.\n") sys.exit(1) p = config.fancy[0] if verbose >= 3: print(f"Fancy: Loading \"{p}\"...") if not os.path.exists(p): if verbose >= 1: sys.stderr.write( f"Error: Could not find fancy disk image plist at \"{p}\"\n") sys.exit(1) try: fancy = plistlib.readPlist(p) except BaseException: if verbose >= 1: sys.stderr.write( f"Error: Could not parse fancy disk image plist at \"{p}\"\n") sys.exit(1) try: assert "window_bounds" not in fancy or ( isinstance( fancy["window_bounds"], list) and len( fancy["window_bounds"]) == 4) assert "background_picture" not in fancy or isinstance( fancy["background_picture"], str) assert "icon_size" not in fancy or isinstance(fancy["icon_size"], int) assert "applications_symlink" not in fancy or isinstance( fancy["applications_symlink"], bool) if "items_position" in fancy: assert isinstance(fancy["items_position"], dict) for key, value in fancy["items_position"].items(): assert isinstance( value, list) and len(value) == 2 and isinstance( value[0], int) and isinstance( value[1], int) except BaseException: if verbose >= 1: sys.stderr.write( f"Error: Bad format of fancy disk image plist at \"{p}\"\n") sys.exit(1) if "background_picture" in fancy: bp = fancy["background_picture"] if verbose >= 3: print(f"Fancy: Resolving background picture \"{bp}\"...") if not os.path.exists(bp): bp = os.path.join(os.path.dirname(p), bp) if not os.path.exists(bp): if verbose >= 1: sys.stderr.write( "Error: Could not find background picture at \"{}\" or \"{}\"\n".format( fancy["background_picture"], bp)) sys.exit(1) else: fancy["background_picture"] = bp else: fancy = None # ------------------------------------------------ if os.path.exists("dist"): if verbose >= 2: print("+ Removing old dist folder +") shutil.rmtree("dist") # ------------------------------------------------ if len(config.volname) == 1: volname = config.volname[0] else: volname = app_bundle_name # ------------------------------------------------ target = os.path.join("dist", "BitcoinABC-Qt.app") if verbose >= 2: print("+ Copying source bundle +") if verbose >= 3: print(app_bundle, "->", target) os.mkdir("dist") shutil.copytree(app_bundle, target, symlinks=True) applicationBundle = ApplicationBundleInfo(target) # ------------------------------------------------ if verbose >= 2: print("+ Deploying frameworks +") try: deploymentInfo = deployFrameworksForAppBundle( applicationBundle, config.strip, verbose) if deploymentInfo.qtPath is None: deploymentInfo.qtPath = os.getenv("QTDIR", None) if deploymentInfo.qtPath is None: if verbose >= 1: sys.stderr.write( "Warning: Could not detect Qt's path, skipping plugin deployment!\n") config.plugins = False except RuntimeError as e: if verbose >= 1: sys.stderr.write(f"Error: {str(e)}\n") sys.exit(1) # ------------------------------------------------ if config.plugins: if verbose >= 2: print("+ Deploying plugins +") try: deployPlugins(applicationBundle, deploymentInfo, config.strip, verbose) except RuntimeError as e: if verbose >= 1: sys.stderr.write(f"Error: {str(e)}\n") sys.exit(1) # ------------------------------------------------ if len(config.add_qt_tr) == 0: add_qt_tr = [] else: if translations_dir is not None: qt_tr_dir = translations_dir else: if deploymentInfo.qtPath is not None: qt_tr_dir = os.path.join(deploymentInfo.qtPath, "translations") else: sys.stderr.write("Error: Could not find Qt translation path\n") sys.exit(1) add_qt_tr = [f"qt_{lng}.qm" for lng in config.add_qt_tr[0].split(",")] for lng_file in add_qt_tr: p = os.path.join(qt_tr_dir, lng_file) if verbose >= 3: print(f"Checking for \"{p}\"...") if not os.path.exists(p): if verbose >= 1: sys.stderr.write( f"Error: Could not find Qt translation file \"{lng_file}\"\n") sys.exit(1) # ------------------------------------------------ if verbose >= 2: print("+ Installing qt.conf +") with open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") as f: f.write(qt_conf.encode()) # ------------------------------------------------ if len(add_qt_tr) > 0 and verbose >= 2: print("+ Adding Qt translations +") for lng_file in add_qt_tr: if verbose >= 3: print( os.path.join( qt_tr_dir, lng_file), "->", os.path.join( applicationBundle.resourcesPath, lng_file)) shutil.copy2( os.path.join( qt_tr_dir, lng_file), os.path.join( applicationBundle.resourcesPath, lng_file)) # ------------------------------------------------ if len(config.add_resources) > 0 and verbose >= 2: print("+ Adding additional resources +") for p in config.add_resources: t = os.path.join(applicationBundle.resourcesPath, os.path.basename(p)) if verbose >= 3: print(p, "->", t) if os.path.isdir(p): shutil.copytree(p, t, symlinks=True) else: shutil.copy2(p, t) # ------------------------------------------------ if config.sign and 'CODESIGNARGS' not in os.environ: print("You must set the CODESIGNARGS environment variable. Skipping signing.") elif config.sign: if verbose >= 1: print(f"Code-signing app bundle {target}") subprocess.check_call( f"codesign --force {os.environ['CODESIGNARGS']} {target}", shell=True) # ------------------------------------------------ if config.dmg is not None: def runHDIUtil(verb: str, image_basename: str, **kwargs) -> int: hdiutil_args = ["hdiutil", verb, image_basename + ".dmg"] if "capture_stdout" in kwargs: del kwargs["capture_stdout"] run = subprocess.check_output else: if verbose < 2: hdiutil_args.append("-quiet") elif verbose >= 3: hdiutil_args.append("-verbose") run = subprocess.check_call for key, value in kwargs.items(): hdiutil_args.append("-" + key) if value is not True: hdiutil_args.append(str(value)) return run(hdiutil_args, universal_newlines=True) if verbose >= 2: if fancy is None: print("+ Creating .dmg disk image +") else: print("+ Preparing .dmg disk image +") if config.dmg != "": dmg_name = config.dmg else: spl = app_bundle_name.split(" ") dmg_name = spl[0] + "".join(p.capitalize() for p in spl[1:]) if fancy is None: try: runHDIUtil( "create", dmg_name, srcfolder="dist", format="UDBZ", volname=volname, ov=True) except subprocess.CalledProcessError as e: sys.exit(e.returncode) else: if verbose >= 3: print("Determining size of \"dist\"...") size = 0 for path, dirs, files in os.walk("dist"): for file in files: size += os.path.getsize(os.path.join(path, file)) size += int(size * 0.15) if verbose >= 3: print("Creating temp image for modification...") try: runHDIUtil( "create", dmg_name + ".temp", srcfolder="dist", format="UDRW", size=size, volname=volname, ov=True) except subprocess.CalledProcessError as e: sys.exit(e.returncode) if verbose >= 3: print("Attaching temp image...") try: output = runHDIUtil( "attach", dmg_name + ".temp", readwrite=True, noverify=True, noautoopen=True, capture_stdout=True) except subprocess.CalledProcessError as e: sys.exit(e.returncode) m = re.search("/Volumes/(.+$)", output) disk_root = m.group(0) disk_name = m.group(1) if verbose >= 2: print("+ Applying fancy settings +") if "background_picture" in fancy: bg_path = os.path.join( disk_root, ".background", os.path.basename( fancy["background_picture"])) os.mkdir(os.path.dirname(bg_path)) if verbose >= 3: print(fancy["background_picture"], "->", bg_path) shutil.copy2(fancy["background_picture"], bg_path) else: bg_path = None if fancy.get("applications_symlink", False): os.symlink( "/Applications", os.path.join( disk_root, "Applications")) # The Python appscript package broke with OSX 10.8 and isn't being fixed. # So we now build up an AppleScript string and use the osascript command # to make the .dmg file pretty: appscript = Template(""" on run argv tell application "Finder" tell disk "$disk" open set current view of container window to icon view set toolbar visible of container window to false set statusbar visible of container window to false set the bounds of container window to {$window_bounds} set theViewOptions to the icon view options of container window set arrangement of theViewOptions to not arranged set icon size of theViewOptions to $icon_size $background_commands $items_positions close -- close/reopen works around a bug... open update without registering applications delay 5 eject end tell end tell end run """) itemscript = Template( 'set position of item "${item}" of container window to {${position}}') items_positions = [] if "items_position" in fancy: for name, position in fancy["items_position"].items(): params = {"item": name, "position": ",".join( [str(p) for p in position])} items_positions.append(itemscript.substitute(params)) params = { "disk": volname, "window_bounds": "300,300,800,620", "icon_size": "96", "background_commands": "", "items_positions": "\n ".join(items_positions) } if "window_bounds" in fancy: params["window_bounds"] = ",".join( [str(p) for p in fancy["window_bounds"]]) if "icon_size" in fancy: params["icon_size"] = str(fancy["icon_size"]) if bg_path is not None: # Set background file, then call SetFile to make it invisible. # (note: making it invisible first makes set background picture fail) bgscript = Template("""set background picture of theViewOptions to file ".background:$bgpic" do shell script "SetFile -a V /Volumes/$disk/.background/$bgpic" """) params["background_commands"] = bgscript.substitute( {"bgpic": os.path.basename(bg_path), "disk": params["disk"]}) s = appscript.substitute(params) if verbose >= 2: print("Running AppleScript:") print(s) p = subprocess.Popen(['osascript', '-'], stdin=subprocess.PIPE) p.communicate(input=s.encode('utf-8')) if p.returncode: print("Error running osascript.") if verbose >= 2: print("+ Finalizing .dmg disk image +") time.sleep(5) try: runHDIUtil( "convert", dmg_name + ".temp", format="UDBZ", o=dmg_name + ".dmg", ov=True) except subprocess.CalledProcessError as e: sys.exit(e.returncode) os.unlink(dmg_name + ".temp.dmg") # ------------------------------------------------ if verbose >= 2: print("+ Done +") sys.exit(0) diff --git a/contrib/tracing/p2p_monitor.py b/contrib/tracing/p2p_monitor.py index 9732d73c4..a46183b2b 100755 --- a/contrib/tracing/p2p_monitor.py +++ b/contrib/tracing/p2p_monitor.py @@ -1,259 +1,259 @@ #!/usr/bin/env python3 """ Interactive bitcoind P2P network traffic monitor utilizing USDT and the net:inbound_message and net:outbound_message tracepoints. """ # This script demonstrates what USDT for Bitcoin ABC can enable. It uses BCC # (https://github.com/iovisor/bcc) to load a sandboxed eBPF program into the # Linux kernel (root privileges are required). The eBPF program attaches to two # statically defined tracepoints. The tracepoint 'net:inbound_message' is called # when a new P2P message is received, and 'net:outbound_message' is called on # outbound P2P messages. The eBPF program submits the P2P messages to # this script via a BPF ring buffer. import curses import sys from curses import panel, wrapper from typing import List from bcc import BPF, USDT # BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into # a sandboxed Linux kernel VM. program = """ #include // Tor v3 addresses are 62 chars + 6 chars for the port (':12345'). // I2P addresses are 60 chars + 6 chars for the port (':12345'). #define MAX_PEER_ADDR_LENGTH 62 + 6 #define MAX_PEER_CONN_TYPE_LENGTH 20 #define MAX_MSG_TYPE_LENGTH 20 struct p2p_message { u64 peer_id; char peer_addr[MAX_PEER_ADDR_LENGTH]; char peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH]; char msg_type[MAX_MSG_TYPE_LENGTH]; u64 msg_size; }; // Two BPF perf buffers for pushing data (here P2P messages) to user space. BPF_PERF_OUTPUT(inbound_messages); BPF_PERF_OUTPUT(outbound_messages); int trace_inbound_message(struct pt_regs *ctx) { struct p2p_message msg = {}; bpf_usdt_readarg(1, ctx, &msg.peer_id); bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); bpf_usdt_readarg(5, ctx, &msg.msg_size); inbound_messages.perf_submit(ctx, &msg, sizeof(msg)); return 0; }; int trace_outbound_message(struct pt_regs *ctx) { struct p2p_message msg = {}; bpf_usdt_readarg(1, ctx, &msg.peer_id); bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); bpf_usdt_readarg(5, ctx, &msg.msg_size); outbound_messages.perf_submit(ctx, &msg, sizeof(msg)); return 0; }; """ class Message: """ A P2P network message. """ msg_type = "" size = 0 data = bytes() inbound = False def __init__(self, msg_type, size, inbound): self.msg_type = msg_type self.size = size self.inbound = inbound class Peer: """ A P2P network peer. """ id = 0 address = "" connection_type = "" last_messages: List[Message] = [] total_inbound_msgs = 0 total_inbound_bytes = 0 total_outbound_msgs = 0 total_outbound_bytes = 0 - def __init__(self, id, address, connection_type): - self.id = id + def __init__(self, peer_id, address, connection_type): + self.id = peer_id self.address = address self.connection_type = connection_type self.last_messages = [] def add_message(self, message): self.last_messages.append(message) if len(self.last_messages) > 25: self.last_messages.pop(0) if message.inbound: self.total_inbound_bytes += message.size self.total_inbound_msgs += 1 else: self.total_outbound_bytes += message.size self.total_outbound_msgs += 1 def main(bitcoind_path): peers = {} bitcoind_with_usdts = USDT(path=str(bitcoind_path)) # attaching the trace functions defined in the BPF program to the # tracepoints bitcoind_with_usdts.enable_probe( probe="inbound_message", fn_name="trace_inbound_message") bitcoind_with_usdts.enable_probe( probe="outbound_message", fn_name="trace_outbound_message") bpf = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) # BCC: perf buffer handle function for inbound_messages def handle_inbound(_, data, size): """ Inbound message handler. Called each time a message is submitted to the inbound_messages BPF table.""" event = bpf["inbound_messages"].event(data) if event.peer_id not in peers: peer = Peer(event.peer_id, event.peer_addr.decode( "utf-8"), event.peer_conn_type.decode("utf-8")) peers[peer.id] = peer peers[event.peer_id].add_message( Message(event.msg_type.decode("utf-8"), event.msg_size, True)) # BCC: perf buffer handle function for outbound_messages def handle_outbound(_, data, size): """ Outbound message handler. Called each time a message is submitted to the outbound_messages BPF table.""" event = bpf["outbound_messages"].event(data) if event.peer_id not in peers: peer = Peer(event.peer_id, event.peer_addr.decode( "utf-8"), event.peer_conn_type.decode("utf-8")) peers[peer.id] = peer peers[event.peer_id].add_message( Message(event.msg_type.decode("utf-8"), event.msg_size, False)) # BCC: add handlers to the inbound and outbound perf buffers bpf["inbound_messages"].open_perf_buffer(handle_inbound) bpf["outbound_messages"].open_perf_buffer(handle_outbound) wrapper(loop, bpf, peers) def loop(screen, bpf, peers): screen.nodelay(1) cur_list_pos = 0 win = curses.newwin(30, 70, 2, 7) win.erase() win.border(ord("|"), ord("|"), ord("-"), ord("-"), ord("-"), ord("-"), ord("-"), ord("-")) info_panel = panel.new_panel(win) info_panel.hide() ROWS_AVALIABLE_FOR_LIST = curses.LINES - 5 scroll = 0 while True: try: # BCC: poll the perf buffers for new events or timeout after 50ms bpf.perf_buffer_poll(timeout=50) ch = screen.getch() if (ch == curses.KEY_DOWN or ch == ord("j")) and cur_list_pos < len( peers.keys()) - 1 and info_panel.hidden(): cur_list_pos += 1 if cur_list_pos >= ROWS_AVALIABLE_FOR_LIST: scroll += 1 if ((ch == curses.KEY_UP or ch == ord("k")) and cur_list_pos > 0 and info_panel.hidden()): cur_list_pos -= 1 if scroll > 0: scroll -= 1 if ch == ord('\n') or ch == ord(' '): if info_panel.hidden(): info_panel.show() else: info_panel.hide() screen.erase() render( screen, peers, cur_list_pos, scroll, ROWS_AVALIABLE_FOR_LIST, info_panel) curses.panel.update_panels() screen.refresh() except KeyboardInterrupt: exit() def render(screen, peers, cur_list_pos, scroll, ROWS_AVALIABLE_FOR_LIST, info_panel): """ renders the list of peers and details panel This code is unrelated to USDT, BCC and BPF. """ header_format = "%6s %-20s %-20s %-22s %-67s" row_format = "%6s %-5d %9d byte %-5d %9d byte %-22s %-67s" screen.addstr(0, 1, (" P2P Message Monitor "), curses.A_REVERSE) screen.addstr( 1, 0, (" Navigate with UP/DOWN or J/K and select a peer with ENTER or SPACE to see individual P2P messages"), curses.A_NORMAL) screen.addstr(3, 0, header_format % ("PEER", "OUTBOUND", "INBOUND", "TYPE", "ADDR"), curses.A_BOLD | curses.A_UNDERLINE) peer_list = sorted(peers.keys())[scroll:ROWS_AVALIABLE_FOR_LIST + scroll] for i, peer_id in enumerate(peer_list): peer = peers[peer_id] screen.addstr(i + 4, 0, row_format % (peer.id, peer.total_outbound_msgs, peer.total_outbound_bytes, peer.total_inbound_msgs, peer.total_inbound_bytes, peer.connection_type, peer.address), curses.A_REVERSE if i + scroll == cur_list_pos else curses.A_NORMAL) if i + scroll == cur_list_pos: info_window = info_panel.window() info_window.erase() info_window.border( ord("|"), ord("|"), ord("-"), ord("-"), ord("-"), ord("-"), ord("-"), ord("-")) info_window.addstr( 1, 1, f"PEER {peer.id} ({peer.address})".center(68), curses.A_REVERSE | curses.A_BOLD) info_window.addstr( 2, 1, f" OUR NODE{peer.connection_type:^54}PEER ", curses.A_BOLD) for i, msg in enumerate(peer.last_messages): if msg.inbound: info_window.addstr( i + 3, 1, f"{f'<--- {msg.msg_type} ({msg.size} bytes) ':68s}", curses.A_NORMAL) else: info_window.addstr( i + 3, 1, f" {msg.msg_type} ({msg.size} byte) --->", curses.A_NORMAL) if __name__ == "__main__": if len(sys.argv) < 2: print("USAGE:", sys.argv[0], "path/to/bitcoind") exit() path = sys.argv[1] main(path)