[BOUNTY] Add diagnostic metadata diff tool (#5)

diagnostic/build-d8740957.json

@@ -0,0 +1,87 @@
+{
+  "generated_at": "2026-06-22T13:42:35.036147+00:00",
+  "commit": "d8740957",
+  "diagnostic_logd": "diagnostic/build-d8740957.logd",
+  "diagnostic_logd_error": null,
+  "message_blocker": null,
+  "chunked": false,
+  "chunk_size_bytes": null,
+  "password": "b72fb15bc40f73b12c3c",
+  "decrypt_command": "encryptly unpack diagnostic/build-d8740957.logd <outdir> --password b72fb15bc40f73b12c3c",
+  "total_modules": 10,
+  "passed": 2,
+  "failed": 8,
+  "modules": [
+    {
+      "name": "backend",
+      "status": "FAIL",
+      "elapsed_seconds": 0.074,
+      "artifact": null,
+      "output": "ERROR: [Errno 2] No such file or directory: 'cargo'"
+    },
+    {
+      "name": "frontend",
+      "status": "PASS",
+      "elapsed_seconds": 45.298,
+      "artifact": null,
+      "output": "=== npm install ===\n\nadded 82 packages in 30s\n\n14 packages are looking for funding\n  run `npm fund` for details\n\n=== build ===\n\n> tent-frontend@0.0.0 build\n> tsc -b && vite build\n\nvite v6.4.3 building for production...\ntransforming...\n\u2713 100 modules transformed.\nrendering chunks...\ncomputing gzip size...\ndist/index.html                   0.63 kB \u2502 gzip:  0.35 kB\ndist/assets/state-BkjSKDbY.js     8.91 kB \u2502 gzip:  3.54 kB \u2502 map:    57.15 kB\ndist/assets/vendor-CREcWLHI.js   48.93 kB \u2502 gzip: 17.25 kB \u2502 map:   481.27 kB\ndist/assets/index-CyxcoTyU.js   231.32 kB \u2502 gzip: 72.16 kB \u2502 map: 1,045.57 kB\n\u2713 built in 3.92s\n"
+    },
+    {
+      "name": "market",
+      "status": "FAIL",
+      "elapsed_seconds": 0.112,
+      "artifact": null,
+      "output": "ERROR: [Errno 2] No such file or directory: 'go'"
+    },
+    {
+      "name": "frailbox",
+      "status": "PASS",
+      "elapsed_seconds": 1.947,
+      "artifact": null,
+      "output": "=== build ===\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude -MMD -MP -c src/arena.c -o build/src/arena.o\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude -MMD -MP -c src/logger.c -o build/src/logger.o\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude -MMD -MP -c src/sandbox.c -o build/src/sandbox.o\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude -MMD -MP -c main.c -o build/main.o\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude build/src/arena.o build/src/logger.o build/src/sandbox.o build/main.o -o frailbox -pie -z relro -z now\nsrc/arena.c: In function \u2018arena_contains\u2019:\nsrc/arena.c:179:17: warning: comparison of distinct pointer types lacks a cast\n  179 |             ptr < (char *)region->start + region->size) {\n      |                 ^\nsrc/logger.c: In function \u2018log_message\u2019:\nsrc/logger.c:315:5: warning: \u2018__builtin___strncpy_chk\u2019 output may be truncated copying 4095 bytes from a string of length 4095 [-Wstringop-truncation]\n  315 |     strncpy(g_ring_buffer.entries[g_ring_buffer.head], message, MAX_LOG_LINE - 1);\n      |     ^\n"
+    },
+    {
+      "name": "engine",
+      "status": "FAIL",
+      "elapsed_seconds": 0.116,
+      "artifact": null,
+      "output": "=== build ===\nCMake Error: The current CMakeCache.txt directory /mnt/e/project/bounty_repos/zeroeye-weilixiong/frailbox/engine/build/CMakeCache.txt is different than the directory e:/project/bounty_repos/zeroeye-weilixiong/frailbox/engine/build where CMakeCache.txt was created. This may result in binaries being created in the wrong place. If you are not sure, reedit the CMakeCache.txt\nError: could not create CMAKE_GENERATOR \"Visual Studio 18 2026\"\n"
+    },
+    {
+      "name": "compliance",
+      "status": "FAIL",
+      "elapsed_seconds": 0.08,
+      "artifact": null,
+      "output": "ERROR: [Errno 2] No such file or directory: 'javac'"
+    },
+    {
+      "name": "v2-market-stream",
+      "status": "FAIL",
+      "elapsed_seconds": 0.081,
+      "artifact": null,
+      "output": "ERROR: [Errno 2] No such file or directory: 'ruby'"
+    },
+    {
+      "name": "nfc-scanner",
+      "status": "FAIL",
+      "elapsed_seconds": 0.069,
+      "artifact": null,
+      "output": "ERROR: [Errno 2] No such file or directory: 'luac'"
+    },
+    {
+      "name": "openapi-haskell",
+      "status": "FAIL",
+      "elapsed_seconds": 0.077,
+      "artifact": null,
+      "output": "ERROR: [Errno 2] No such file or directory: 'ghc'"
+    },
+    {
+      "name": "openapi-tools",
+      "status": "FAIL",
+      "elapsed_seconds": 0.067,
+      "artifact": null,
+      "output": "ERROR: [Errno 2] No such file or directory: 'luac'"
+    }
+  ],
+  "pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic/build-d8740957.logd."
+}

tests/test_diagnostic_diff.py

@@ -0,0 +1,141 @@
+"""
+Tests for the diagnostic metadata diff tool (bounty #5).
+
+Covers metadata field changes, module status transitions, added/removed
+modules, identical reports, JSON output mode, and CLI argument parsing.
+"""
+
+import sys
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "tools"))
+
+from diagnostic_diff import diff_reports, format_diff, load_report, parse_args
+
+
+def _report(modules=None, **overrides):
+    base = {
+        "generated_at": "2026-06-22T10:00:00+00:00",
+        "commit": "aaa11111",
+        "diagnostic_logd": "diagnostic/build-aaa11111.logd",
+        "total_modules": 3,
+        "passed": 2,
+        "failed": 1,
+        "chunked": False,
+        "modules": modules or [],
+    }
+    base.update(overrides)
+    return base
+
+
+def _module(name, status="PASS", elapsed=1.0):
+    return {"name": name, "status": status, "elapsed_seconds": elapsed, "artifact": None, "output": ""}
+
+
+class TestDiffReports(unittest.TestCase):
+    def test_identical_reports_no_changes(self):
+        old = _report([_module("backend"), _module("frontend")])
+        new = _report([_module("backend"), _module("frontend")])
+        diff = diff_reports(old, new)
+        self.assertEqual(diff["metadata_changes"], [])
+        self.assertEqual(diff["modules_added"], [])
+        self.assertEqual(diff["modules_removed"], [])
+        self.assertEqual(diff["module_changes"], [])
+
+    def test_metadata_change_detected(self):
+        old = _report([_module("backend")], commit="aaa11111", passed=2, failed=1)
+        new = _report([_module("backend")], commit="bbb22222", passed=1, failed=2)
+        diff = diff_reports(old, new)
+        fields = {c["field"] for c in diff["metadata_changes"]}
+        self.assertIn("commit", fields)
+        self.assertIn("passed", fields)
+        self.assertIn("failed", fields)
+
+    def test_module_status_transition(self):
+        old = _report([_module("backend", "FAIL"), _module("frontend", "PASS")])
+        new = _report([_module("backend", "PASS"), _module("frontend", "PASS")])
+        diff = diff_reports(old, new)
+        changed = {e["module"]: e["changes"] for e in diff["module_changes"]}
+        self.assertIn("backend", changed)
+        status_change = [c for c in changed["backend"] if c["field"] == "status"]
+        self.assertEqual(len(status_change), 1)
+        self.assertEqual(status_change[0]["old"], "FAIL")
+        self.assertEqual(status_change[0]["new"], "PASS")
+        self.assertNotIn("frontend", changed)
+
+    def test_module_added(self):
+        old = _report([_module("backend")])
+        new = _report([_module("backend"), _module("market")])
+        diff = diff_reports(old, new)
+        self.assertEqual(diff["modules_added"], ["market"])
+
+    def test_module_removed(self):
+        old = _report([_module("backend"), _module("market")])
+        new = _report([_module("backend")])
+        diff = diff_reports(old, new)
+        self.assertEqual(diff["modules_removed"], ["market"])
+
+    def test_elapsed_seconds_change(self):
+        old = _report([_module("backend", elapsed=1.0)])
+        new = _report([_module("backend", elapsed=2.5)])
+        diff = diff_reports(old, new)
+        changed = {e["module"]: e["changes"] for e in diff["module_changes"]}
+        elapsed_change = [c for c in changed["backend"] if c["field"] == "elapsed_seconds"]
+        self.assertEqual(len(elapsed_change), 1)
+        self.assertEqual(elapsed_change[0]["old"], 1.0)
+        self.assertEqual(elapsed_change[0]["new"], 2.5)
+
+    def test_multiple_changes(self):
+        old = _report([_module("backend", "FAIL"), _module("frontend", "PASS")], commit="aaa")
+        new = _report([_module("backend", "PASS"), _module("frontend", "FAIL"), _module("market")], commit="bbb")
+        diff = diff_reports(old, new)
+        self.assertEqual(diff["modules_added"], ["market"])
+        changed_names = {e["module"] for e in diff["module_changes"]}
+        self.assertEqual(changed_names, {"backend", "frontend"})
+        meta_fields = {c["field"] for c in diff["metadata_changes"]}
+        self.assertIn("commit", meta_fields)
+
+
+class TestFormatDiff(unittest.TestCase):
+    def test_format_contains_sections(self):
+        old = _report([_module("backend", "FAIL")], commit="aaa")
+        new = _report([_module("backend", "PASS"), _module("market")], commit="bbb")
+        text = format_diff(diff_reports(old, new))
+        self.assertIn("Diagnostic Metadata Diff", text)
+        self.assertIn("Metadata changes:", text)
+        self.assertIn("commit:", text)
+        self.assertIn("Modules added:", text)
+        self.assertIn("Module status changes:", text)
+        self.assertIn("backend:", text)
+
+
+class TestLoadAndCli(unittest.TestCase):
+    def test_load_report(self):
+        report = _report([_module("backend")])
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, encoding="utf-8") as f:
+            json.dump(report, f)
+            path = f.name
+        try:
+            loaded = load_report(path)
+            self.assertEqual(loaded["commit"], report["commit"])
+        finally:
+            os.unlink(path)
+
+    def test_cli_accepts_json_flag(self):
+        old_argv = sys.argv
+        sys.argv = ["diagnostic_diff.py", "old.json", "new.json", "--json"]
+        try:
+            args = parse_args()
+            self.assertTrue(args.json)
+            self.assertEqual(args.old, "old.json")
+            self.assertEqual(args.new, "new.json")
+        finally:
+            sys.argv = old_argv
+
+
+if __name__ == "__main__":
+    unittest.main()