Changeset View
Changeset View
Standalone View
Standalone View
test/functional/test_runner.py
| Show First 20 Lines • Show All 148 Lines • ▼ Show 20 Lines | def run(self, portseed_offset): | ||||
| if process.returncode == TEST_EXIT_PASSED and stderr == "": | if process.returncode == TEST_EXIT_PASSED and stderr == "": | ||||
| status = "Passed" | status = "Passed" | ||||
| elif process.returncode == TEST_EXIT_SKIPPED: | elif process.returncode == TEST_EXIT_SKIPPED: | ||||
| status = "Skipped" | status = "Skipped" | ||||
| else: | else: | ||||
| status = "Failed" | status = "Failed" | ||||
| return TestResult(self.test_num, self.test_case, testdir, status, | return TestResult(self.test_num, self.test_case, testdir, status, | ||||
| int(time.time() - time0), stdout, stderr) | time.time() - time0, stdout, stderr) | ||||
| def on_ci(): | def on_ci(): | ||||
| return os.getenv('TRAVIS') == 'true' or os.getenv( | return os.getenv('TRAVIS') == 'true' or os.getenv( | ||||
| 'TEAMCITY_VERSION') is not None | 'TEAMCITY_VERSION') is not None | ||||
| def main(): | def main(): | ||||
| ▲ Show 20 Lines • Show All 226 Lines • ▼ Show 20 Lines | if len(test_list) > 1 and num_jobs > 1: | ||||
| except subprocess.CalledProcessError as e: | except subprocess.CalledProcessError as e: | ||||
| sys.stdout.buffer.write(e.output) | sys.stdout.buffer.write(e.output) | ||||
| raise | raise | ||||
| # Run Tests | # Run Tests | ||||
| time0 = time.time() | time0 = time.time() | ||||
| test_results = execute_test_processes( | test_results = execute_test_processes( | ||||
| num_jobs, test_list, tests_dir, tmpdir, flags, failfast) | num_jobs, test_list, tests_dir, tmpdir, flags, failfast) | ||||
| runtime = int(time.time() - time0) | runtime = time.time() - time0 | ||||
| max_len_name = len(max(test_list, key=len)) | max_len_name = len(max(test_list, key=len)) | ||||
| print_results(test_results, tests_dir, max_len_name, | print_results(test_results, tests_dir, max_len_name, | ||||
| runtime, combined_logs_len) | runtime, combined_logs_len) | ||||
| save_results_as_junit(test_results, junitoutput, runtime, test_suite_name) | save_results_as_junit(test_results, junitoutput, runtime, test_suite_name) | ||||
| if (build_timings is not None): | if (build_timings is not None): | ||||
| build_timings.save_timings(test_results) | build_timings.save_timings(test_results) | ||||
| ▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | def handle_message(message, running_jobs): | ||||
| if isinstance(message, TestResult): | if isinstance(message, TestResult): | ||||
| test_result = message | test_result = message | ||||
| running_jobs.remove((test_result.num, test_result.name)) | running_jobs.remove((test_result.num, test_result.name)) | ||||
| test_results.append(test_result) | test_results.append(test_result) | ||||
| if test_result.status == "Passed": | if test_result.status == "Passed": | ||||
| print("{}{}{} passed, Duration: {} s".format( | print("{}{}{} passed, Duration: {} s".format( | ||||
| BOLD[1], test_result.name, BOLD[0], test_result.time)) | BOLD[1], test_result.name, BOLD[0], TimeResolution.seconds(test_result.time))) | ||||
| elif test_result.status == "Skipped": | elif test_result.status == "Skipped": | ||||
| print("{}{}{} skipped".format( | print("{}{}{} skipped".format( | ||||
| BOLD[1], test_result.name, BOLD[0])) | BOLD[1], test_result.name, BOLD[0])) | ||||
| else: | else: | ||||
| print("{}{}{} failed, Duration: {} s\n".format( | print("{}{}{} failed, Duration: {} s\n".format( | ||||
| BOLD[1], test_result.name, BOLD[0], test_result.time)) | BOLD[1], test_result.name, BOLD[0], TimeResolution.seconds(test_result.time))) | ||||
| print(BOLD[1] + 'stdout:' + BOLD[0]) | print(BOLD[1] + 'stdout:' + BOLD[0]) | ||||
| print(test_result.stdout) | print(test_result.stdout) | ||||
| print(BOLD[1] + 'stderr:' + BOLD[0]) | print(BOLD[1] + 'stderr:' + BOLD[0]) | ||||
| print(test_result.stderr) | print(test_result.stderr) | ||||
| if failfast: | if failfast: | ||||
| logging.debug("Early exiting after test failure") | logging.debug("Early exiting after test failure") | ||||
| failfast_event.set() | failfast_event.set() | ||||
| ▲ Show 20 Lines • Show All 117 Lines • ▼ Show 20 Lines | for test_result in test_results: | ||||
| deque( | deque( | ||||
| combined_logs.splitlines(), | combined_logs.splitlines(), | ||||
| combined_logs_len))) | combined_logs_len))) | ||||
| status = TICK + "Passed" if all_passed else CROSS + "Failed" | status = TICK + "Passed" if all_passed else CROSS + "Failed" | ||||
| if not all_passed: | if not all_passed: | ||||
| results += RED[1] | results += RED[1] | ||||
| results += BOLD[1] + "\n{} | {} | {} s (accumulated) \n".format( | results += BOLD[1] + "\n{} | {} | {} s (accumulated) \n".format( | ||||
| "ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0] | "ALL".ljust(max_len_name), status.ljust(9), TimeResolution.seconds(time_sum)) + BOLD[0] | ||||
| if not all_passed: | if not all_passed: | ||||
| results += RED[0] | results += RED[0] | ||||
| results += "Runtime: {} s\n".format(runtime) | results += "Runtime: {} s\n".format(TimeResolution.seconds(runtime)) | ||||
| print(results) | print(results) | ||||
| class TestResult(): | class TestResult(): | ||||
| """ | """ | ||||
| Simple data structure to store test result values and print them properly | Simple data structure to store test result values and print them properly | ||||
| """ | """ | ||||
| Show All 22 Lines | def __repr__(self): | ||||
| elif self.status == "Failed": | elif self.status == "Failed": | ||||
| color = RED | color = RED | ||||
| glyph = CROSS | glyph = CROSS | ||||
| elif self.status == "Skipped": | elif self.status == "Skipped": | ||||
| color = GREY | color = GREY | ||||
| glyph = CIRCLE | glyph = CIRCLE | ||||
| return color[1] + "{} | {}{} | {} s\n".format( | return color[1] + "{} | {}{} | {} s\n".format( | ||||
| self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0] | self.name.ljust(self.padding), glyph, self.status.ljust(7), TimeResolution.seconds(self.time)) + color[0] | ||||
| @property | @property | ||||
| def was_successful(self): | def was_successful(self): | ||||
| return self.status != "Failed" | return self.status != "Failed" | ||||
| def get_all_scripts_from_disk(test_dir, non_scripts): | def get_all_scripts_from_disk(test_dir, non_scripts): | ||||
| """ | """ | ||||
| ▲ Show 20 Lines • Show All 140 Lines • ▼ Show 20 Lines | def save_results_as_junit(test_results, file_name, time, test_suite_name): | ||||
| """ | """ | ||||
| e_test_suite = ET.Element("testsuite", | e_test_suite = ET.Element("testsuite", | ||||
| {"name": "{}".format(test_suite_name), | {"name": "{}".format(test_suite_name), | ||||
| "tests": str(len(test_results)), | "tests": str(len(test_results)), | ||||
| # "errors": | # "errors": | ||||
| "failures": str(len([t for t in test_results if t.status == "Failed"])), | "failures": str(len([t for t in test_results if t.status == "Failed"])), | ||||
| "id": "0", | "id": "0", | ||||
| "skipped": str(len([t for t in test_results if t.status == "Skipped"])), | "skipped": str(len([t for t in test_results if t.status == "Skipped"])), | ||||
| "time": str(time), | "time": str(TimeResolution.milliseconds(time)), | ||||
| "timestamp": datetime.datetime.now().isoformat('T') | "timestamp": datetime.datetime.now().isoformat('T') | ||||
| }) | }) | ||||
| for test_result in test_results: | for test_result in test_results: | ||||
| e_test_case = ET.SubElement(e_test_suite, "testcase", | e_test_case = ET.SubElement(e_test_suite, "testcase", | ||||
| {"name": test_result.name, | {"name": test_result.name, | ||||
| "classname": test_result.name, | "classname": test_result.name, | ||||
| "time": str(test_result.time) | "time": str(TimeResolution.milliseconds(test_result.time)) | ||||
| } | } | ||||
| ) | ) | ||||
| if test_result.status == "Skipped": | if test_result.status == "Skipped": | ||||
| ET.SubElement(e_test_case, "skipped") | ET.SubElement(e_test_case, "skipped") | ||||
| elif test_result.status == "Failed": | elif test_result.status == "Failed": | ||||
| ET.SubElement(e_test_case, "failure") | ET.SubElement(e_test_case, "failure") | ||||
| # no special element for passed tests | # no special element for passed tests | ||||
| Show All 38 Lines | def get_merged_timings(self, new_timings): | ||||
| merged = list(merged.values()) | merged = list(merged.values()) | ||||
| merged.sort(key=lambda t, key=key: t[key]) | merged.sort(key=lambda t, key=key: t[key]) | ||||
| return merged | return merged | ||||
| def save_timings(self, test_results): | def save_timings(self, test_results): | ||||
| # we only save test that have passed - timings for failed test might be | # we only save test that have passed - timings for failed test might be | ||||
| # wrong (timeouts or early fails) | # wrong (timeouts or early fails) | ||||
| passed_results = [t for t in test_results if t.status == 'Passed'] | passed_results = [t for t in test_results if t.status == 'Passed'] | ||||
| new_timings = list(map(lambda t: {'name': t.name, 'time': t.time}, | new_timings = list(map(lambda t: {'name': t.name, 'time': TimeResolution.seconds(t.time)}, | ||||
| passed_results)) | passed_results)) | ||||
| merged_timings = self.get_merged_timings(new_timings) | merged_timings = self.get_merged_timings(new_timings) | ||||
| with open(self.timing_file, 'w', encoding="utf8") as f: | with open(self.timing_file, 'w', encoding="utf8") as f: | ||||
| json.dump(merged_timings, f, indent=True) | json.dump(merged_timings, f, indent=True) | ||||
| class TimeResolution: | |||||
| @staticmethod | |||||
| def seconds(time_fractional_second): | |||||
| return round(time_fractional_second) | |||||
| @staticmethod | |||||
| def milliseconds(time_fractional_second): | |||||
| return round(time_fractional_second, 3) | |||||
| if __name__ == '__main__': | if __name__ == '__main__': | ||||
| main() | main() | ||||