sausage/userland/diffutils/util/compare_test_results.py

#!/usr/bin/env python3

"""
Compare the current GNU test results to the last results gathered from the main branch to
highlight if a PR is making the results better/worse.
Don't exit with error code if all failing tests are in the ignore-intermittent.txt list.

Exit status: 0 if the comparison shows no new failure, 1 if new non-intermittent
failures appeared, and 2 if the comparison couldn't be performed at all (e.g. one
of the result files is missing or malformed).
"""

import json
import sys
import argparse
from pathlib import Path


def load_ignore_list(ignore_file):
    """Load list of intermittent test names to ignore from file."""
    ignore_set = set()
    if ignore_file and Path(ignore_file).exists():
        with open(ignore_file, "r") as f:
            for line in f:
                line = line.strip()
                if line and not line.startswith("#"):
                    ignore_set.add(line)
    return ignore_set


def extract_test_results(json_data):
    """Extract test results from a diffutils test-results.json.

    Note: unlike sed, diffutils JSON has no 'summary' object — results are
    computed from the 'tests' array using the 'result' and 'test' fields.
    """
    tests = json_data.get("tests", [])
    passed  = sum(1 for t in tests if t.get("result") == "PASS")
    failed  = sum(1 for t in tests if t.get("result") == "FAIL")
    skipped = sum(1 for t in tests if t.get("result") == "SKIP")
    summary = {"total": len(tests), "passed": passed, "failed": failed, "skipped": skipped}
    failed_tests = [t["test"] for t in tests if t.get("result") == "FAIL"]
    return summary, failed_tests


def compare_results(current_file, reference_file, ignore_file=None, output_file=None):
    """Compare current results with reference results."""
    ignore_set = load_ignore_list(ignore_file)

    try:
        with open(current_file, "r") as f:
            current_data = json.load(f)
        current_summary, current_failed = extract_test_results(current_data)
    except Exception as e:
        print(f"Error loading current results: {e}")
        return 2

    try:
        with open(reference_file, "r") as f:
            reference_data = json.load(f)
        reference_summary, reference_failed = extract_test_results(reference_data)
    except Exception as e:
        print(f"Error loading reference results: {e}")
        return 2

    # Calculate differences
    pass_diff  = int(current_summary.get("passed",  0)) - int(reference_summary.get("passed",  0))
    fail_diff  = int(current_summary.get("failed",  0)) - int(reference_summary.get("failed",  0))
    total_diff = int(current_summary.get("total",   0)) - int(reference_summary.get("total",   0))

    # Find new failures and improvements
    current_failed_set   = set(current_failed)
    reference_failed_set = set(reference_failed)

    new_failures  = current_failed_set   - reference_failed_set
    improvements  = reference_failed_set - current_failed_set

    # Filter out intermittent failures
    non_intermittent_new_failures = new_failures - ignore_set

    # Check if results are identical (no changes)
    no_changes = (
        pass_diff == 0
        and fail_diff == 0
        and total_diff == 0
        and not new_failures
        and not improvements
    )

    # If no changes, write empty output to prevent comment posting
    if no_changes:
        if output_file:
            with open(output_file, "w") as f:
                f.write("")
        return 0

    # Prepare output message
    output_lines = []

    output_lines.append("Test results comparison:")
    output_lines.append(
        f"  Current:   TOTAL: {current_summary.get('total', 0)} / PASSED: {current_summary.get('passed', 0)} / FAILED: {current_summary.get('failed', 0)} / SKIPPED: {current_summary.get('skipped', 0)}"
    )
    output_lines.append(
        f"  Reference: TOTAL: {reference_summary.get('total', 0)} / PASSED: {reference_summary.get('passed', 0)} / FAILED: {reference_summary.get('failed', 0)} / SKIPPED: {reference_summary.get('skipped', 0)}"
    )
    output_lines.append("")

    if pass_diff != 0 or fail_diff != 0 or total_diff != 0:
        output_lines.append("Changes from main branch:")
        output_lines.append(f"  TOTAL: {total_diff:+d}")
        output_lines.append(f"  PASSED: {pass_diff:+d}")
        output_lines.append(f"  FAILED: {fail_diff:+d}")
        output_lines.append("")

    if new_failures:
        output_lines.append(f"New test failures ({len(new_failures)}):")
        for test in sorted(new_failures):
            if test in ignore_set:
                output_lines.append(f"  - {test} (intermittent)")
            else:
                output_lines.append(f"  - {test}")
        output_lines.append("")

    if improvements:
        output_lines.append(f"Test improvements ({len(improvements)}):")
        for test in sorted(improvements):
            output_lines.append(f"  + {test}")
        output_lines.append("")

    output_text = "\n".join(output_lines)
    if output_file:
        with open(output_file, "w") as f:
            f.write(output_text)
    else:
        print(output_text)

    if non_intermittent_new_failures:
        print(
            f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures"
        )
        return 1

    return 0


def main():
    parser = argparse.ArgumentParser(description="Compare GNU diffutils test results")
    parser.add_argument("current",   help="Current test results JSON file")
    parser.add_argument("reference", help="Reference test results JSON file")
    parser.add_argument(
        "--ignore-file", help="File containing intermittent test names to ignore"
    )
    parser.add_argument("--output",  help="Output file for comparison results")

    args = parser.parse_args()

    return compare_results(args.current, args.reference, args.ignore_file, args.output)


if __name__ == "__main__":
    sys.exit(main())