Changeset View
Changeset View
Standalone View
Standalone View
test/functional/combine_logs.py
| #!/usr/bin/env python3 | #!/usr/bin/env python3 | ||||
| """Combine logs from multiple bitcoin nodes as well as the test_framework log. | """Combine logs from multiple bitcoin nodes as well as the test_framework log. | ||||
| This streams the combined log output to stdout. Use combine_logs.py > outputfile | This streams the combined log output to stdout. Use combine_logs.py > outputfile | ||||
| to write to an outputfile.""" | to write to an outputfile. | ||||
| If no argument is provided, the most recent test directory will be used.""" | |||||
| import argparse | import argparse | ||||
| from collections import defaultdict, namedtuple | from collections import defaultdict, namedtuple | ||||
| import glob | import glob | ||||
| import heapq | import heapq | ||||
| import itertools | import itertools | ||||
| import os | import os | ||||
| import re | import re | ||||
| import sys | import sys | ||||
| import tempfile | |||||
| # N.B.: don't import any local modules here - this script must remain executable | |||||
| # without the parent module installed. | |||||
| # Should match same symbol in `test_framework.test_framework`. | |||||
| TMPDIR_PREFIX = "bitcoin_func_test_" | |||||
| # Matches on the date format at the start of the log event | # Matches on the date format at the start of the log event | ||||
| TIMESTAMP_PATTERN = re.compile( | TIMESTAMP_PATTERN = re.compile( | ||||
| r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?Z") | r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?Z") | ||||
| LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event']) | LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event']) | ||||
| def main(): | def main(): | ||||
| """Main function. Parses args, reads the log files and renders them as text or html.""" | """Main function. Parses args, reads the log files and renders them as text or html.""" | ||||
| parser = argparse.ArgumentParser( | parser = argparse.ArgumentParser( | ||||
| usage='%(prog)s [options] <test temporary directory>', description=__doc__) | description=__doc__, formatter_class=argparse.RawTextHelpFormatter) | ||||
| parser.add_argument( | |||||
| 'testdir', nargs='?', default='', | |||||
| help='temporary test directory to combine logs from. ' | |||||
| 'Defaults to the most recent') | |||||
| parser.add_argument('-c', '--color', dest='color', action='store_true', | parser.add_argument('-c', '--color', dest='color', action='store_true', | ||||
| help='outputs the combined log with events colored by source (requires posix terminal colors. Use less -r for viewing)') | help='outputs the combined log with events colored by ' | ||||
| 'source (requires posix terminal colors. Use less' | |||||
| ' -r for viewing)') | |||||
| parser.add_argument('--html', dest='html', action='store_true', | parser.add_argument('--html', dest='html', action='store_true', | ||||
| help='outputs the combined log as html. Requires jinja2. pip install jinja2') | help='outputs the combined log as html. ' | ||||
| args, unknown_args = parser.parse_known_args() | 'Requires jinja2. pip install jinja2') | ||||
| args = parser.parse_args() | |||||
| if args.html and args.color: | if args.html and args.color: | ||||
| print("Only one out of --color or --html should be specified") | print("Only one out of --color or --html should be specified") | ||||
| sys.exit(1) | sys.exit(1) | ||||
| # There should only be one unknown argument - the path of the temporary | testdir = args.testdir or find_latest_test_dir() | ||||
| # test directory | |||||
| if len(unknown_args) != 1: | if not testdir: | ||||
| print("Unexpected arguments" + str(unknown_args)) | print("No test directories found") | ||||
| sys.exit(1) | sys.exit(1) | ||||
| log_events = read_logs(unknown_args[0]) | if not args.testdir: | ||||
| print("Opening latest test directory: {}".format(testdir), | |||||
| file=sys.stderr) | |||||
| log_events = read_logs(testdir) | |||||
| print_logs(log_events, color=args.color, html=args.html) | print_logs(log_events, color=args.color, html=args.html) | ||||
| def read_logs(tmp_dir): | def read_logs(tmp_dir): | ||||
| """Reads log files. | """Reads log files. | ||||
| Delegates to generator function get_log_events() to provide individual log events | Delegates to generator function get_log_events() to provide individual log events | ||||
| Show All 16 Lines | for i in itertools.count(): | ||||
| logfile = "{}/node{}/{}/debug.log".format(tmp_dir, i, chain) | logfile = "{}/node{}/{}/debug.log".format(tmp_dir, i, chain) | ||||
| if not os.path.isfile(logfile): | if not os.path.isfile(logfile): | ||||
| break | break | ||||
| files.append(("node{}".format(i), logfile)) | files.append(("node{}".format(i), logfile)) | ||||
| return heapq.merge(*[get_log_events(source, f) for source, f in files]) | return heapq.merge(*[get_log_events(source, f) for source, f in files]) | ||||
| def find_latest_test_dir(): | |||||
| """Returns the latest tmpfile test directory prefix.""" | |||||
| tmpdir = tempfile.gettempdir() | |||||
| def join_tmp(basename): | |||||
| return os.path.join(tmpdir, basename) | |||||
| def is_valid_test_tmpdir(basename): | |||||
| fullpath = join_tmp(basename) | |||||
| return ( | |||||
| os.path.isdir(fullpath) | |||||
| and basename.startswith(TMPDIR_PREFIX) | |||||
| and os.access(fullpath, os.R_OK) | |||||
| ) | |||||
| testdir_paths = [join_tmp(name) for name in os.listdir(tmpdir) | |||||
| if is_valid_test_tmpdir(name)] | |||||
| return max(testdir_paths, key=os.path.getmtime) if testdir_paths else None | |||||
| def get_log_events(source, logfile): | def get_log_events(source, logfile): | ||||
| """Generator function that returns individual log events. | """Generator function that returns individual log events. | ||||
| Log events may be split over multiple lines. We use the timestamp | Log events may be split over multiple lines. We use the timestamp | ||||
| regex match as the marker for a new log event.""" | regex match as the marker for a new log event.""" | ||||
| try: | try: | ||||
| with open(logfile, 'r', encoding='utf-8') as infile: | with open(logfile, 'r', encoding='utf-8') as infile: | ||||
| event = '' | event = '' | ||||
| ▲ Show 20 Lines • Show All 65 Lines • Show Last 20 Lines | |||||