Changeset View
Changeset View
Standalone View
Standalone View
test/functional/combine_logs.py
| #!/usr/bin/env python3 | #!/usr/bin/env python3 | ||||
| """Combine logs from multiple bitcoin nodes as well as the test_framework log. | """Combine logs from multiple bitcoin nodes as well as the test_framework log. | ||||
| This streams the combined log output to stdout. Use combine_logs.py > outputfile | This streams the combined log output to stdout. Use combine_logs.py > outputfile | ||||
| to write to an outputfile.""" | to write to an outputfile.""" | ||||
| import argparse | import argparse | ||||
| from collections import defaultdict, namedtuple | from collections import defaultdict, namedtuple | ||||
| import glob | import glob | ||||
| import heapq | import heapq | ||||
| import itertools | import itertools | ||||
| import os | import os | ||||
| import re | import re | ||||
| import sys | import sys | ||||
| # Matches on the date format at the start of the log event | # Matches on the date format at the start of the log event | ||||
| TIMESTAMP_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z") | TIMESTAMP_PATTERN = re.compile( | ||||
| r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?Z") | |||||
| LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event']) | LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event']) | ||||
| def main(): | def main(): | ||||
| """Main function. Parses args, reads the log files and renders them as text or html.""" | """Main function. Parses args, reads the log files and renders them as text or html.""" | ||||
| parser = argparse.ArgumentParser( | parser = argparse.ArgumentParser( | ||||
| ▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines | try: | ||||
| if line == '\n': | if line == '\n': | ||||
| continue | continue | ||||
| # if this line has a timestamp, it's the start of a new log | # if this line has a timestamp, it's the start of a new log | ||||
| # event. | # event. | ||||
| time_match = TIMESTAMP_PATTERN.match(line) | time_match = TIMESTAMP_PATTERN.match(line) | ||||
| if time_match: | if time_match: | ||||
| if event: | if event: | ||||
| yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip()) | yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip()) | ||||
| event = line | |||||
| timestamp = time_match.group() | timestamp = time_match.group() | ||||
| if time_match.group(1) is None: | |||||
| # timestamp does not have microseconds. Add zeroes. | |||||
| timestamp_micro = timestamp.replace("Z", ".000000Z") | |||||
| line = line.replace(timestamp, timestamp_micro) | |||||
| timestamp = timestamp_micro | |||||
| event = line | |||||
| # if it doesn't have a timestamp, it's a continuation line of | # if it doesn't have a timestamp, it's a continuation line of | ||||
| # the previous log. | # the previous log. | ||||
| else: | else: | ||||
| event += "\n" + line | # Add the line. Prefix with space equivalent to the source | ||||
| # + timestamp so log lines are aligned | |||||
| event += " " + line | |||||
| # Flush the final event | # Flush the final event | ||||
| yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip()) | yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip()) | ||||
| except FileNotFoundError: | except FileNotFoundError: | ||||
| print("File {} could not be opened. Continuing without it.".format( | print("File {} could not be opened. Continuing without it.".format( | ||||
| logfile), file=sys.stderr) | logfile), file=sys.stderr) | ||||
| def print_logs(log_events, color=False, html=False): | def print_logs(log_events, color=False, html=False): | ||||
| """Renders the iterator of log events into text or html.""" | """Renders the iterator of log events into text or html.""" | ||||
| if not html: | if not html: | ||||
| colors = defaultdict(lambda: '') | colors = defaultdict(lambda: '') | ||||
| if color: | if color: | ||||
| colors["test"] = "\033[0;36m" # CYAN | colors["test"] = "\033[0;36m" # CYAN | ||||
| colors["node0"] = "\033[0;34m" # BLUE | colors["node0"] = "\033[0;34m" # BLUE | ||||
| colors["node1"] = "\033[0;32m" # GREEN | colors["node1"] = "\033[0;32m" # GREEN | ||||
| colors["node2"] = "\033[0;31m" # RED | colors["node2"] = "\033[0;31m" # RED | ||||
| colors["node3"] = "\033[0;33m" # YELLOW | colors["node3"] = "\033[0;33m" # YELLOW | ||||
| colors["reset"] = "\033[0m" # Reset font color | colors["reset"] = "\033[0m" # Reset font color | ||||
| for event in log_events: | for event in log_events: | ||||
| lines = event.event.splitlines() | |||||
| print("{0} {1: <5} {2} {3}".format( | print("{0} {1: <5} {2} {3}".format( | ||||
| colors[event.source.rstrip()], event.source, event.event, colors["reset"])) | colors[event.source.rstrip()], event.source, lines[0], colors["reset"])) | ||||
| if len(lines) > 1: | |||||
| for line in lines[1:]: | |||||
| print("{0}{1}{2}".format( | |||||
| colors[event.source.rstrip()], line, colors["reset"])) | |||||
| else: | else: | ||||
| try: | try: | ||||
| import jinja2 | import jinja2 | ||||
| except ImportError: | except ImportError: | ||||
| print("jinja2 not found. Try `pip install jinja2`") | print("jinja2 not found. Try `pip install jinja2`") | ||||
| sys.exit(1) | sys.exit(1) | ||||
| print(jinja2.Environment(loader=jinja2.FileSystemLoader('./')) | print(jinja2.Environment(loader=jinja2.FileSystemLoader('./')) | ||||
| .get_template('combined_log_template.html') | .get_template('combined_log_template.html') | ||||
| .render(title="Combined Logs from testcase", log_events=[event._asdict() for event in log_events])) | .render(title="Combined Logs from testcase", log_events=[event._asdict() for event in log_events])) | ||||
| if __name__ == '__main__': | if __name__ == '__main__': | ||||
| main() | main() | ||||