#!/usr/bin/env python3 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Script to generate an HTML report for flaky tests detected in JUnit XML results. This script processes JUnit test results generated by Flank and Firebase Test Lab, specifically looking for tests marked with the flaky="true" attribute. It creates a styled HTML report with links to Firebase Test Lab for detailed test analysis. - Searches for the FullJUnitReport.xml file in the results directory. - Identifies test cases with flaky="true" attribute. - Extracts failure information and Firebase Test Lab web links. - Generates a formatted HTML report with clickable links to test executions. - Only creates the report if flaky tests are found. - Designed for use in Taskcluster following a Firebase Test Lab test execution. Flank: https://flank.github.io/flank/ Usage: python3 generate-flaky-report-from-ftl.py --results python3 generate-flaky-report-from-ftl.py --results --output """ import argparse import logging import sys from pathlib import Path from typing import Any from junitparser import Attr, JUnitXml, TestCase class test_case(TestCase): flaky = Attr() def setup_logging(): """Configure logging for the script.""" log_format = "%(levelname)s: %(message)s" logging.basicConfig(level=logging.INFO, format=log_format) def find_junit_xml_files(results_dir: Path) -> list[Path]: """Find the FullJUnitReport.xml file in the results directory. Args: results_dir: Path to the results directory Returns: List containing path to FullJUnitReport.xml if it exists """ full_report = results_dir / "FullJUnitReport.xml" if full_report.exists(): logging.info(f"Found FullJUnitReport.xml in {results_dir}") return [full_report] else: logging.info(f"FullJUnitReport.xml not found in {results_dir}") return [] def extract_flaky_tests(xml_files: list[Path]) -> list[dict[str, Any]]: """Parse JUnit XML files and extract tests marked as flaky. Args: xml_files: List of paths to JUnit XML files Returns: List of dictionaries containing flaky test information """ flaky_tests = [] for xml_file in xml_files: try: xml = JUnitXml.fromfile(str(xml_file)) for suite in xml: for case in suite: # Use custom test_case class to access flaky attribute cur_case = test_case.fromelem(case) if isinstance(cur_case, TestCase) and cur_case.flaky: flaky_info = { "name": cur_case.name, "classname": cur_case.classname, "time": cur_case.time, "file": xml_file.name, } # Extract webLink if present web_link = cur_case._elem.find("webLink") if web_link is not None and web_link.text: flaky_info["weblink"] = web_link.text # Capture failure/error information if present if cur_case.result: result_info = [] for result in cur_case.result: result_info.append( { "type": ( result.type if result.type else "Failure Stack Trace" ), "message": ( result.message if result.message else result.text ), } ) flaky_info["results"] = result_info flaky_tests.append(flaky_info) logging.info( f"Found flaky test: {cur_case.classname}.{cur_case.name}" ) except Exception as e: logging.error(f"Failed to parse {xml_file}: {e}") continue return flaky_tests def generate_html_report(flaky_tests: list[dict[str, Any]], output_path: Path) -> None: """Generate an HTML report for flaky tests. Args: flaky_tests: List of flaky test information output_path: Path where the HTML report should be written """ html_content = f""" Flaky Test Report

⚠️ Flaky Test Report

Total Flaky Tests Found: {len(flaky_tests)}
""" for test in flaky_tests: html_content += f"""
{test['name']}
Class: {test['classname']}
Execution Time: {test.get('time', 'N/A')}s | Source: {test['file']}
""" if "weblink" in test: html_content += f""" View in Firebase Test Lab """ if "results" in test: for result in test["results"]: html_content += f"""
{result.get('type', 'Unknown')}
{result.get('message', 'No message available')}
""" html_content += """
""" html_content += """
Generated by Mozilla Mobile Test Engineering
""" output_path.write_text(html_content, encoding="utf-8") logging.info(f"HTML flaky report written to {output_path}") def main(): """Parse arguments and generate flaky test report.""" parser = argparse.ArgumentParser( description="Generate an HTML report for flaky tests from JUnit XML results" ) parser.add_argument( "--results", required=True, help="Path to the results directory containing JUnit XML files", ) parser.add_argument( "--output", default="/builds/worker/artifacts/HtmlFlakyReport.html", help="Path where the HTML report should be written", ) args = parser.parse_args() results_dir = Path(args.results) output_path = Path(args.output) if not results_dir.exists(): logging.warning(f"Results directory does not exist: {results_dir}") sys.exit(0) # Find all JUnit XML files xml_files = find_junit_xml_files(results_dir) if not xml_files: logging.info("No JUnit XML files found, skipping flaky report generation") sys.exit(0) # Extract flaky tests flaky_tests = extract_flaky_tests(xml_files) if not flaky_tests: logging.info("No flaky tests detected, skipping report generation") sys.exit(0) # Generate HTML report output_path.parent.mkdir(parents=True, exist_ok=True) generate_html_report(flaky_tests, output_path) logging.info( f"Flaky test report generation complete: {len(flaky_tests)} flaky tests found" ) sys.exit(0) if __name__ == "__main__": setup_logging() main()