diff --git a/diagnostic/build-d8740957.json b/diagnostic/build-d8740957.json new file mode 100644 index 00000000..fe415077 --- /dev/null +++ b/diagnostic/build-d8740957.json @@ -0,0 +1,87 @@ +{ + "generated_at": "2026-06-22T13:42:35.036147+00:00", + "commit": "d8740957", + "diagnostic_logd": "diagnostic/build-d8740957.logd", + "diagnostic_logd_error": null, + "message_blocker": null, + "chunked": false, + "chunk_size_bytes": null, + "password": "b72fb15bc40f73b12c3c", + "decrypt_command": "encryptly unpack diagnostic/build-d8740957.logd --password b72fb15bc40f73b12c3c", + "total_modules": 10, + "passed": 2, + "failed": 8, + "modules": [ + { + "name": "backend", + "status": "FAIL", + "elapsed_seconds": 0.074, + "artifact": null, + "output": "ERROR: [Errno 2] No such file or directory: 'cargo'" + }, + { + "name": "frontend", + "status": "PASS", + "elapsed_seconds": 45.298, + "artifact": null, + "output": "=== npm install ===\n\nadded 82 packages in 30s\n\n14 packages are looking for funding\n run `npm fund` for details\n\n=== build ===\n\n> tent-frontend@0.0.0 build\n> tsc -b && vite build\n\nvite v6.4.3 building for production...\ntransforming...\n\u2713 100 modules transformed.\nrendering chunks...\ncomputing gzip size...\ndist/index.html 0.63 kB \u2502 gzip: 0.35 kB\ndist/assets/state-BkjSKDbY.js 8.91 kB \u2502 gzip: 3.54 kB \u2502 map: 57.15 kB\ndist/assets/vendor-CREcWLHI.js 48.93 kB \u2502 gzip: 17.25 kB \u2502 map: 481.27 kB\ndist/assets/index-CyxcoTyU.js 231.32 kB \u2502 gzip: 72.16 kB \u2502 map: 1,045.57 kB\n\u2713 built in 3.92s\n" + }, + { + "name": "market", + "status": "FAIL", + "elapsed_seconds": 0.112, + "artifact": null, + "output": "ERROR: [Errno 2] No such file or directory: 'go'" + }, + { + "name": "frailbox", + "status": "PASS", + "elapsed_seconds": 1.947, + "artifact": null, + "output": "=== build ===\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude -MMD -MP -c src/arena.c -o build/src/arena.o\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude -MMD -MP -c src/logger.c -o build/src/logger.o\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude -MMD -MP -c src/sandbox.c -o build/src/sandbox.o\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude -MMD -MP -c main.c -o build/main.o\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude build/src/arena.o build/src/logger.o build/src/sandbox.o build/main.o -o frailbox -pie -z relro -z now\nsrc/arena.c: In function \u2018arena_contains\u2019:\nsrc/arena.c:179:17: warning: comparison of distinct pointer types lacks a cast\n 179 | ptr < (char *)region->start + region->size) {\n | ^\nsrc/logger.c: In function \u2018log_message\u2019:\nsrc/logger.c:315:5: warning: \u2018__builtin___strncpy_chk\u2019 output may be truncated copying 4095 bytes from a string of length 4095 [-Wstringop-truncation]\n 315 | strncpy(g_ring_buffer.entries[g_ring_buffer.head], message, MAX_LOG_LINE - 1);\n | ^\n" + }, + { + "name": "engine", + "status": "FAIL", + "elapsed_seconds": 0.116, + "artifact": null, + "output": "=== build ===\nCMake Error: The current CMakeCache.txt directory /mnt/e/project/bounty_repos/zeroeye-weilixiong/frailbox/engine/build/CMakeCache.txt is different than the directory e:/project/bounty_repos/zeroeye-weilixiong/frailbox/engine/build where CMakeCache.txt was created. This may result in binaries being created in the wrong place. If you are not sure, reedit the CMakeCache.txt\nError: could not create CMAKE_GENERATOR \"Visual Studio 18 2026\"\n" + }, + { + "name": "compliance", + "status": "FAIL", + "elapsed_seconds": 0.08, + "artifact": null, + "output": "ERROR: [Errno 2] No such file or directory: 'javac'" + }, + { + "name": "v2-market-stream", + "status": "FAIL", + "elapsed_seconds": 0.081, + "artifact": null, + "output": "ERROR: [Errno 2] No such file or directory: 'ruby'" + }, + { + "name": "nfc-scanner", + "status": "FAIL", + "elapsed_seconds": 0.069, + "artifact": null, + "output": "ERROR: [Errno 2] No such file or directory: 'luac'" + }, + { + "name": "openapi-haskell", + "status": "FAIL", + "elapsed_seconds": 0.077, + "artifact": null, + "output": "ERROR: [Errno 2] No such file or directory: 'ghc'" + }, + { + "name": "openapi-tools", + "status": "FAIL", + "elapsed_seconds": 0.067, + "artifact": null, + "output": "ERROR: [Errno 2] No such file or directory: 'luac'" + } + ], + "pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic/build-d8740957.logd." +} \ No newline at end of file diff --git a/diagnostic/build-d8740957.logd b/diagnostic/build-d8740957.logd new file mode 100644 index 00000000..596009f0 Binary files /dev/null and b/diagnostic/build-d8740957.logd differ diff --git a/tests/test_diagnostic_diff.py b/tests/test_diagnostic_diff.py new file mode 100644 index 00000000..5e3774a0 --- /dev/null +++ b/tests/test_diagnostic_diff.py @@ -0,0 +1,141 @@ +""" +Tests for the diagnostic metadata diff tool (bounty #5). + +Covers metadata field changes, module status transitions, added/removed +modules, identical reports, JSON output mode, and CLI argument parsing. +""" + +import sys +import json +import os +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "tools")) + +from diagnostic_diff import diff_reports, format_diff, load_report, parse_args + + +def _report(modules=None, **overrides): + base = { + "generated_at": "2026-06-22T10:00:00+00:00", + "commit": "aaa11111", + "diagnostic_logd": "diagnostic/build-aaa11111.logd", + "total_modules": 3, + "passed": 2, + "failed": 1, + "chunked": False, + "modules": modules or [], + } + base.update(overrides) + return base + + +def _module(name, status="PASS", elapsed=1.0): + return {"name": name, "status": status, "elapsed_seconds": elapsed, "artifact": None, "output": ""} + + +class TestDiffReports(unittest.TestCase): + def test_identical_reports_no_changes(self): + old = _report([_module("backend"), _module("frontend")]) + new = _report([_module("backend"), _module("frontend")]) + diff = diff_reports(old, new) + self.assertEqual(diff["metadata_changes"], []) + self.assertEqual(diff["modules_added"], []) + self.assertEqual(diff["modules_removed"], []) + self.assertEqual(diff["module_changes"], []) + + def test_metadata_change_detected(self): + old = _report([_module("backend")], commit="aaa11111", passed=2, failed=1) + new = _report([_module("backend")], commit="bbb22222", passed=1, failed=2) + diff = diff_reports(old, new) + fields = {c["field"] for c in diff["metadata_changes"]} + self.assertIn("commit", fields) + self.assertIn("passed", fields) + self.assertIn("failed", fields) + + def test_module_status_transition(self): + old = _report([_module("backend", "FAIL"), _module("frontend", "PASS")]) + new = _report([_module("backend", "PASS"), _module("frontend", "PASS")]) + diff = diff_reports(old, new) + changed = {e["module"]: e["changes"] for e in diff["module_changes"]} + self.assertIn("backend", changed) + status_change = [c for c in changed["backend"] if c["field"] == "status"] + self.assertEqual(len(status_change), 1) + self.assertEqual(status_change[0]["old"], "FAIL") + self.assertEqual(status_change[0]["new"], "PASS") + self.assertNotIn("frontend", changed) + + def test_module_added(self): + old = _report([_module("backend")]) + new = _report([_module("backend"), _module("market")]) + diff = diff_reports(old, new) + self.assertEqual(diff["modules_added"], ["market"]) + + def test_module_removed(self): + old = _report([_module("backend"), _module("market")]) + new = _report([_module("backend")]) + diff = diff_reports(old, new) + self.assertEqual(diff["modules_removed"], ["market"]) + + def test_elapsed_seconds_change(self): + old = _report([_module("backend", elapsed=1.0)]) + new = _report([_module("backend", elapsed=2.5)]) + diff = diff_reports(old, new) + changed = {e["module"]: e["changes"] for e in diff["module_changes"]} + elapsed_change = [c for c in changed["backend"] if c["field"] == "elapsed_seconds"] + self.assertEqual(len(elapsed_change), 1) + self.assertEqual(elapsed_change[0]["old"], 1.0) + self.assertEqual(elapsed_change[0]["new"], 2.5) + + def test_multiple_changes(self): + old = _report([_module("backend", "FAIL"), _module("frontend", "PASS")], commit="aaa") + new = _report([_module("backend", "PASS"), _module("frontend", "FAIL"), _module("market")], commit="bbb") + diff = diff_reports(old, new) + self.assertEqual(diff["modules_added"], ["market"]) + changed_names = {e["module"] for e in diff["module_changes"]} + self.assertEqual(changed_names, {"backend", "frontend"}) + meta_fields = {c["field"] for c in diff["metadata_changes"]} + self.assertIn("commit", meta_fields) + + +class TestFormatDiff(unittest.TestCase): + def test_format_contains_sections(self): + old = _report([_module("backend", "FAIL")], commit="aaa") + new = _report([_module("backend", "PASS"), _module("market")], commit="bbb") + text = format_diff(diff_reports(old, new)) + self.assertIn("Diagnostic Metadata Diff", text) + self.assertIn("Metadata changes:", text) + self.assertIn("commit:", text) + self.assertIn("Modules added:", text) + self.assertIn("Module status changes:", text) + self.assertIn("backend:", text) + + +class TestLoadAndCli(unittest.TestCase): + def test_load_report(self): + report = _report([_module("backend")]) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, encoding="utf-8") as f: + json.dump(report, f) + path = f.name + try: + loaded = load_report(path) + self.assertEqual(loaded["commit"], report["commit"]) + finally: + os.unlink(path) + + def test_cli_accepts_json_flag(self): + old_argv = sys.argv + sys.argv = ["diagnostic_diff.py", "old.json", "new.json", "--json"] + try: + args = parse_args() + self.assertTrue(args.json) + self.assertEqual(args.old, "old.json") + self.assertEqual(args.new, "new.json") + finally: + sys.argv = old_argv + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/diagnostic_diff.py b/tools/diagnostic_diff.py new file mode 100644 index 00000000..65ce9ab5 --- /dev/null +++ b/tools/diagnostic_diff.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +""" +Diagnostic metadata diff tool. + +Compares two diagnostic build JSON artifacts (diagnostic/build-*.json) +and reports differences in build metadata and per-module status. Useful +for tracking how a build's outcome changes between commits, environments, +or toolchain upgrades. + +Usage: + python3 tools/diagnostic_diff.py old.json new.json + python3 tools/diagnostic_diff.py old.json new.json --json + python3 tools/diagnostic_diff.py old.json new.json --output diff.json +""" + +import argparse +import json +import sys +from typing import Any, Dict, List, Optional + + +# Top-level fields compared for value changes (everything except large +# free-text fields that are better reported at module granularity). +META_FIELDS = ( + "generated_at", + "commit", + "diagnostic_logd", + "total_modules", + "passed", + "failed", + "chunked", + "chunk_size_bytes", + "message_blocker", + "diagnostic_logd_error", +) + +MODULE_FIELDS = ("status", "elapsed_seconds", "artifact") + + +def load_report(path: str) -> Dict[str, Any]: + """Load a diagnostic build JSON report from ``path``.""" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _module_map(report: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + """Index a report's modules list by module name.""" + modules: Dict[str, Dict[str, Any]] = {} + for module in report.get("modules", []): + name = module.get("name") + if name is not None: + modules[name] = module + return modules + + +def diff_reports(old: Dict[str, Any], new: Dict[str, Any]) -> Dict[str, Any]: + """Compute the metadata diff between two diagnostic reports. + + Returns a dict with ``metadata_changes``, ``modules_added``, + ``modules_removed``, and ``module_changes``. + """ + result: Dict[str, Any] = { + "metadata_changes": [], + "modules_added": [], + "modules_removed": [], + "module_changes": [], + } + + # Top-level metadata value changes. + for field in META_FIELDS: + old_val = old.get(field) + new_val = new.get(field) + if old_val != new_val: + result["metadata_changes"].append( + {"field": field, "old": old_val, "new": new_val} + ) + + old_modules = _module_map(old) + new_modules = _module_map(new) + + result["modules_added"] = sorted(set(new_modules) - set(old_modules)) + result["modules_removed"] = sorted(set(old_modules) - set(new_modules)) + + for name in sorted(set(old_modules) & set(new_modules)): + old_mod = old_modules[name] + new_mod = new_modules[name] + field_changes: List[Dict[str, Any]] = [] + for field in MODULE_FIELDS: + old_val = old_mod.get(field) + new_val = new_mod.get(field) + if old_val != new_val: + field_changes.append( + {"field": field, "old": old_val, "new": new_val} + ) + if field_changes: + result["module_changes"].append( + {"module": name, "changes": field_changes} + ) + + return result + + +def format_diff(diff: Dict[str, Any]) -> str: + """Render a diff dict as human-readable text.""" + lines: List[str] = [] + lines.append("=" * 60) + lines.append("Diagnostic Metadata Diff") + lines.append("=" * 60) + + meta = diff.get("metadata_changes", []) + if meta: + lines.append("\nMetadata changes:") + for change in meta: + lines.append( + f" {change['field']}: {change['old']!r} -> {change['new']!r}" + ) + else: + lines.append("\nMetadata: no changes") + + added = diff.get("modules_added", []) + if added: + lines.append("\nModules added:") + for name in added: + lines.append(f" + {name}") + + removed = diff.get("modules_removed", []) + if removed: + lines.append("\nModules removed:") + for name in removed: + lines.append(f" - {name}") + + module_changes = diff.get("module_changes", []) + if module_changes: + lines.append("\nModule status changes:") + for entry in module_changes: + lines.append(f" {entry['module']}:") + for change in entry["changes"]: + lines.append( + f" {change['field']}: {change['old']!r} -> {change['new']!r}" + ) + else: + lines.append("\nModule status: no changes") + + lines.append("") + return "\n".join(lines) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Diff two diagnostic build JSON reports") + parser.add_argument("old", help="Path to the older diagnostic JSON report") + parser.add_argument("new", help="Path to the newer diagnostic JSON report") + parser.add_argument("--json", "-j", action="store_true", help="Output diff as JSON") + parser.add_argument("--output", "-o", help="Write diff to a file instead of stdout") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + old = load_report(args.old) + new = load_report(args.new) + diff = diff_reports(old, new) + + if args.json: + output = json.dumps(diff, indent=2, default=str) + else: + output = format_diff(diff) + + if args.output: + with open(args.output, "w", encoding="utf-8") as f: + f.write(output) + print(f"Diff written to {args.output}") + else: + print(output) + + return 0 + + +if __name__ == "__main__": + main()