userland: search and exec tools, spawn library (T14)
rg (ripgrep 15.2), grep (ours, on ripgrep's regex crates), find and xargs (ours), sed (uutils 0.3 wrapper), diff and cmp (uutils diffutils 0.6 with -r), fd 10.5, jaq 2.3, env and timeout. userland/spawn is a small Command-like client of the process interface. search-test.nu: 42 cases match native tools.
This commit is contained in:
+162
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Compare the current GNU test results to the last results gathered from the main branch to
|
||||
highlight if a PR is making the results better/worse.
|
||||
Don't exit with error code if all failing tests are in the ignore-intermittent.txt list.
|
||||
|
||||
Exit status: 0 if the comparison shows no new failure, 1 if new non-intermittent
|
||||
failures appeared, and 2 if the comparison couldn't be performed at all (e.g. one
|
||||
of the result files is missing or malformed).
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_ignore_list(ignore_file):
|
||||
"""Load list of intermittent test names to ignore from file."""
|
||||
ignore_set = set()
|
||||
if ignore_file and Path(ignore_file).exists():
|
||||
with open(ignore_file, "r") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
ignore_set.add(line)
|
||||
return ignore_set
|
||||
|
||||
|
||||
def extract_test_results(json_data):
|
||||
"""Extract test results from a diffutils test-results.json.
|
||||
|
||||
Note: unlike sed, diffutils JSON has no 'summary' object — results are
|
||||
computed from the 'tests' array using the 'result' and 'test' fields.
|
||||
"""
|
||||
tests = json_data.get("tests", [])
|
||||
passed = sum(1 for t in tests if t.get("result") == "PASS")
|
||||
failed = sum(1 for t in tests if t.get("result") == "FAIL")
|
||||
skipped = sum(1 for t in tests if t.get("result") == "SKIP")
|
||||
summary = {"total": len(tests), "passed": passed, "failed": failed, "skipped": skipped}
|
||||
failed_tests = [t["test"] for t in tests if t.get("result") == "FAIL"]
|
||||
return summary, failed_tests
|
||||
|
||||
|
||||
def compare_results(current_file, reference_file, ignore_file=None, output_file=None):
|
||||
"""Compare current results with reference results."""
|
||||
ignore_set = load_ignore_list(ignore_file)
|
||||
|
||||
try:
|
||||
with open(current_file, "r") as f:
|
||||
current_data = json.load(f)
|
||||
current_summary, current_failed = extract_test_results(current_data)
|
||||
except Exception as e:
|
||||
print(f"Error loading current results: {e}")
|
||||
return 2
|
||||
|
||||
try:
|
||||
with open(reference_file, "r") as f:
|
||||
reference_data = json.load(f)
|
||||
reference_summary, reference_failed = extract_test_results(reference_data)
|
||||
except Exception as e:
|
||||
print(f"Error loading reference results: {e}")
|
||||
return 2
|
||||
|
||||
# Calculate differences
|
||||
pass_diff = int(current_summary.get("passed", 0)) - int(reference_summary.get("passed", 0))
|
||||
fail_diff = int(current_summary.get("failed", 0)) - int(reference_summary.get("failed", 0))
|
||||
total_diff = int(current_summary.get("total", 0)) - int(reference_summary.get("total", 0))
|
||||
|
||||
# Find new failures and improvements
|
||||
current_failed_set = set(current_failed)
|
||||
reference_failed_set = set(reference_failed)
|
||||
|
||||
new_failures = current_failed_set - reference_failed_set
|
||||
improvements = reference_failed_set - current_failed_set
|
||||
|
||||
# Filter out intermittent failures
|
||||
non_intermittent_new_failures = new_failures - ignore_set
|
||||
|
||||
# Check if results are identical (no changes)
|
||||
no_changes = (
|
||||
pass_diff == 0
|
||||
and fail_diff == 0
|
||||
and total_diff == 0
|
||||
and not new_failures
|
||||
and not improvements
|
||||
)
|
||||
|
||||
# If no changes, write empty output to prevent comment posting
|
||||
if no_changes:
|
||||
if output_file:
|
||||
with open(output_file, "w") as f:
|
||||
f.write("")
|
||||
return 0
|
||||
|
||||
# Prepare output message
|
||||
output_lines = []
|
||||
|
||||
output_lines.append("Test results comparison:")
|
||||
output_lines.append(
|
||||
f" Current: TOTAL: {current_summary.get('total', 0)} / PASSED: {current_summary.get('passed', 0)} / FAILED: {current_summary.get('failed', 0)} / SKIPPED: {current_summary.get('skipped', 0)}"
|
||||
)
|
||||
output_lines.append(
|
||||
f" Reference: TOTAL: {reference_summary.get('total', 0)} / PASSED: {reference_summary.get('passed', 0)} / FAILED: {reference_summary.get('failed', 0)} / SKIPPED: {reference_summary.get('skipped', 0)}"
|
||||
)
|
||||
output_lines.append("")
|
||||
|
||||
if pass_diff != 0 or fail_diff != 0 or total_diff != 0:
|
||||
output_lines.append("Changes from main branch:")
|
||||
output_lines.append(f" TOTAL: {total_diff:+d}")
|
||||
output_lines.append(f" PASSED: {pass_diff:+d}")
|
||||
output_lines.append(f" FAILED: {fail_diff:+d}")
|
||||
output_lines.append("")
|
||||
|
||||
if new_failures:
|
||||
output_lines.append(f"New test failures ({len(new_failures)}):")
|
||||
for test in sorted(new_failures):
|
||||
if test in ignore_set:
|
||||
output_lines.append(f" - {test} (intermittent)")
|
||||
else:
|
||||
output_lines.append(f" - {test}")
|
||||
output_lines.append("")
|
||||
|
||||
if improvements:
|
||||
output_lines.append(f"Test improvements ({len(improvements)}):")
|
||||
for test in sorted(improvements):
|
||||
output_lines.append(f" + {test}")
|
||||
output_lines.append("")
|
||||
|
||||
output_text = "\n".join(output_lines)
|
||||
if output_file:
|
||||
with open(output_file, "w") as f:
|
||||
f.write(output_text)
|
||||
else:
|
||||
print(output_text)
|
||||
|
||||
if non_intermittent_new_failures:
|
||||
print(
|
||||
f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures"
|
||||
)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Compare GNU diffutils test results")
|
||||
parser.add_argument("current", help="Current test results JSON file")
|
||||
parser.add_argument("reference", help="Reference test results JSON file")
|
||||
parser.add_argument(
|
||||
"--ignore-file", help="File containing intermittent test names to ignore"
|
||||
)
|
||||
parser.add_argument("--output", help="Output file for comparison results")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
return compare_results(args.current, args.reference, args.ignore_file, args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user