Changeset View
Changeset View
Standalone View
Standalone View
test/functional/combine_logs.py
| #!/usr/bin/env python3 | #!/usr/bin/env python3 | ||||
| """Combine logs from multiple bitcoin nodes as well as the test_framework log. | """Combine logs from multiple bitcoin nodes as well as the test_framework log. | ||||
| This streams the combined log output to stdout. Use combine_logs.py > outputfile | This streams the combined log output to stdout. Use combine_logs.py > outputfile | ||||
| to write to an outputfile. | to write to an outputfile. | ||||
| If no argument is provided, the most recent test directory will be used.""" | If no argument is provided, the most recent test directory will be used.""" | ||||
| import argparse | import argparse | ||||
| from collections import defaultdict, namedtuple | from collections import defaultdict, namedtuple | ||||
| import glob | import glob | ||||
| import heapq | import heapq | ||||
| import itertools | import itertools | ||||
| import os | import os | ||||
| import pathlib | |||||
| import re | import re | ||||
| import sys | import sys | ||||
| import tempfile | import tempfile | ||||
| # N.B.: don't import any local modules here - this script must remain executable | # N.B.: don't import any local modules here - this script must remain executable | ||||
| # without the parent module installed. | # without the parent module installed. | ||||
| # Should match same symbol in `test_framework.test_framework`. | # Should match same symbol in `test_framework.test_framework`. | ||||
| Show All 32 Lines | def main(): | ||||
| if not testdir: | if not testdir: | ||||
| print("No test directories found") | print("No test directories found") | ||||
| sys.exit(1) | sys.exit(1) | ||||
| if not args.testdir: | if not args.testdir: | ||||
| print("Opening latest test directory: {}".format(testdir), | print("Opening latest test directory: {}".format(testdir), | ||||
| file=sys.stderr) | file=sys.stderr) | ||||
| colors = defaultdict(lambda: '') | |||||
| if args.color: | |||||
| colors["test"] = "\033[0;36m" # CYAN | |||||
| colors["node0"] = "\033[0;34m" # BLUE | |||||
| colors["node1"] = "\033[0;32m" # GREEN | |||||
| colors["node2"] = "\033[0;31m" # RED | |||||
| colors["node3"] = "\033[0;33m" # YELLOW | |||||
| colors["reset"] = "\033[0m" # Reset font color | |||||
| log_events = read_logs(testdir) | log_events = read_logs(testdir) | ||||
| print_logs(log_events, color=args.color, html=args.html) | if args.html: | ||||
| print_logs_html(log_events) | |||||
| else: | |||||
| print_logs_plain(log_events, colors) | |||||
| print_node_warnings(testdir, colors) | |||||
| def read_logs(tmp_dir): | def read_logs(tmp_dir): | ||||
| """Reads log files. | """Reads log files. | ||||
| Delegates to generator function get_log_events() to provide individual log events | Delegates to generator function get_log_events() to provide individual log events | ||||
| for each of the input log files.""" | for each of the input log files.""" | ||||
| Show All 14 Lines | for i in itertools.count(): | ||||
| logfile = "{}/node{}/{}/debug.log".format(tmp_dir, i, chain) | logfile = "{}/node{}/{}/debug.log".format(tmp_dir, i, chain) | ||||
| if not os.path.isfile(logfile): | if not os.path.isfile(logfile): | ||||
| break | break | ||||
| files.append(("node{}".format(i), logfile)) | files.append(("node{}".format(i), logfile)) | ||||
| return heapq.merge(*[get_log_events(source, f) for source, f in files]) | return heapq.merge(*[get_log_events(source, f) for source, f in files]) | ||||
| def print_node_warnings(tmp_dir, colors): | |||||
| """Print nodes' errors and warnings""" | |||||
| warnings = [] | |||||
| for stream in ['stdout', 'stderr']: | |||||
| for i in itertools.count(): | |||||
| folder = "{}/node{}/{}".format(tmp_dir, i, stream) | |||||
| if not os.path.isdir(folder): | |||||
| break | |||||
| for (_, _, fns) in os.walk(folder): | |||||
| for fn in fns: | |||||
| warning = pathlib.Path( | |||||
| '{}/{}'.format(folder, fn)).read_text().strip() | |||||
| if warning: | |||||
| warnings.append(("node{} {}".format(i, stream), | |||||
| warning)) | |||||
| print() | |||||
| for w in warnings: | |||||
| print("{} {} {} {}".format(colors[w[0].split()[0]], | |||||
| w[0], w[1], colors["reset"])) | |||||
| def find_latest_test_dir(): | def find_latest_test_dir(): | ||||
| """Returns the latest tmpfile test directory prefix.""" | """Returns the latest tmpfile test directory prefix.""" | ||||
| tmpdir = tempfile.gettempdir() | tmpdir = tempfile.gettempdir() | ||||
| def join_tmp(basename): | def join_tmp(basename): | ||||
| return os.path.join(tmpdir, basename) | return os.path.join(tmpdir, basename) | ||||
| def is_valid_test_tmpdir(basename): | def is_valid_test_tmpdir(basename): | ||||
| ▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | try: | ||||
| event += " " + line | event += " " + line | ||||
| # Flush the final event | # Flush the final event | ||||
| yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip()) | yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip()) | ||||
| except FileNotFoundError: | except FileNotFoundError: | ||||
| print("File {} could not be opened. Continuing without it.".format( | print("File {} could not be opened. Continuing without it.".format( | ||||
| logfile), file=sys.stderr) | logfile), file=sys.stderr) | ||||
| def print_logs(log_events, color=False, html=False): | def print_logs_plain(log_events, colors): | ||||
| """Renders the iterator of log events into text or html.""" | """Renders the iterator of log events into text.""" | ||||
| if not html: | |||||
| colors = defaultdict(lambda: '') | |||||
| if color: | |||||
| colors["test"] = "\033[0;36m" # CYAN | |||||
| colors["node0"] = "\033[0;34m" # BLUE | |||||
| colors["node1"] = "\033[0;32m" # GREEN | |||||
| colors["node2"] = "\033[0;31m" # RED | |||||
| colors["node3"] = "\033[0;33m" # YELLOW | |||||
| colors["reset"] = "\033[0m" # Reset font color | |||||
| for event in log_events: | for event in log_events: | ||||
| lines = event.event.splitlines() | lines = event.event.splitlines() | ||||
| print("{0} {1: <5} {2} {3}".format( | print("{0} {1: <5} {2} {3}".format(colors[event.source.rstrip()], | ||||
| colors[event.source.rstrip()], event.source, lines[0], colors["reset"])) | event.source, lines[0], | ||||
| colors["reset"])) | |||||
| if len(lines) > 1: | if len(lines) > 1: | ||||
| for line in lines[1:]: | for line in lines[1:]: | ||||
| print("{0}{1}{2}".format( | print("{0}{1}{2}".format( | ||||
| colors[event.source.rstrip()], line, colors["reset"])) | colors[event.source.rstrip()], line, colors["reset"])) | ||||
| else: | |||||
| def print_logs_html(log_events): | |||||
| """Renders the iterator of log events into html.""" | |||||
| try: | try: | ||||
| import jinja2 | import jinja2 | ||||
| except ImportError: | except ImportError: | ||||
| print("jinja2 not found. Try `pip install jinja2`") | print("jinja2 not found. Try `pip install jinja2`") | ||||
| sys.exit(1) | sys.exit(1) | ||||
| print(jinja2.Environment(loader=jinja2.FileSystemLoader('./')) | print(jinja2.Environment(loader=jinja2.FileSystemLoader('./')) | ||||
| .get_template('combined_log_template.html') | .get_template('combined_log_template.html') | ||||
| .render(title="Combined Logs from testcase", log_events=[event._asdict() for event in log_events])) | .render(title="Combined Logs from testcase", | ||||
| log_events=[event._asdict() for event in log_events])) | |||||
| if __name__ == '__main__': | if __name__ == '__main__': | ||||
| main() | main() | ||||