From e9b752ead57ca0c830f5fa2643754f2404aa6039 Mon Sep 17 00:00:00 2001
From: Your Name <ankit.rana1@1mg.com>
Date: Tue, 25 Aug 2026 12:55:08 +0530
Subject: [PATCH] Add DeputyDev v1 offline benchmark results

---
 offline/README.md                             |    1 +
 offline/analysis/benchmark_dashboard.html     |   10 +-
 offline/analysis/benchmark_dashboard.json     | 3975 ++++++++++++-----
 offline/analysis/benchmark_dashboard.py       |    2 +
 .../step1_download_prs.py                     |    2 +-
 offline/results/benchmark_data.json           | 1952 ++++++++
 offline/results/gpt-5.5/candidates.json       | 2217 +++++++++
 offline/results/gpt-5.5/dedup_groups.json     |  812 ++++
 offline/results/gpt-5.5/evaluations.json      | 2624 +++++++++++
 offline/tests/test_step1.py                   |   26 +
 10 files changed, 10491 insertions(+), 1130 deletions(-)
 create mode 100644 offline/results/gpt-5.5/candidates.json
 create mode 100644 offline/results/gpt-5.5/dedup_groups.json
 create mode 100644 offline/results/gpt-5.5/evaluations.json

diff --git a/offline/README.md b/offline/README.md
index 78b2815..ae08f41 100644
--- a/offline/README.md
+++ b/offline/README.md
@@ -13,6 +13,7 @@ Open replication of the code review benchmark used by companies like [Augment](h
 | [CodeRabbit](https://www.coderabbit.ai/) | AI code review |
 | [Cursor Bugbot](https://cursor.com) | AI code review |
 | [Cubic](https://cubic.dev/) | AI code review |
+| [DeputyDev](https://deputydev.ai/) | AI code review |
 | [Devin](https://devin.ai/) | AI assistant |
 | [Gemini](https://gemini.google.com/) | AI assistant |
 | [GitHub Copilot](https://github.com/features/copilot) | AI code review |
diff --git a/offline/analysis/benchmark_dashboard.html b/offline/analysis/benchmark_dashboard.html
index 927d654..dafe2e8 100644
--- a/offline/analysis/benchmark_dashboard.html
+++ b/offline/analysis/benchmark_dashboard.html
@@ -201,6 +201,7 @@
         <div class="predefined-filter" data-filter-id="tool_codeant-v2_pr_size_small_change_type_performance" onclick="applyPredefinedFilter('tool_codeant-v2_pr_size_small_change_type_performance')">Small PRs + Performance Optimization (Precision) <span class="arrow">↗</span></div>
         <div class="predefined-filter" data-filter-id="tool_gitlab_change_type_bug_fix_context_cross_file" onclick="applyPredefinedFilter('tool_gitlab_change_type_bug_fix_context_cross_file')">Bug Fixes + Cross-File <span class="arrow">↗</span></div>
         <div class="predefined-filter" data-filter-id="tool_baz_language_java_domain_authentication" onclick="applyPredefinedFilter('tool_baz_language_java_domain_authentication')">Java + Authentication <span class="arrow">↗</span></div>
+        <div class="predefined-filter" data-filter-id="tool_deputydev-v1_risk_high_context_file" onclick="applyPredefinedFilter('tool_deputydev-v1_risk_high_context_file')">High Risk + File Context <span class="arrow">↗</span></div>
         <div class="predefined-filter" data-filter-id="python_medium" onclick="applyPredefinedFilter('python_medium')">Best for Medium Python PRs <span class="arrow">↗</span></div>
         <div class="predefined-filter" data-filter-id="domain_concurrency" onclick="applyPredefinedFilter('domain_concurrency')">Best for Concurrency <span class="arrow">↗</span></div>
         <div class="predefined-filter" data-filter-id="tool_claude_change_type_bug_fix" onclick="applyPredefinedFilter('tool_claude_change_type_bug_fix')">Best for Bug Fixes <span class="arrow">↗</span></div>
@@ -251,6 +252,7 @@
                 <select id="model-select" onchange="changeModel(this.value)">
                     <option value="anthropic_claude-opus-4-5-20251101" selected>Anthropic claude-opus-4-5-20251101</option>
                     <option value="anthropic_claude-sonnet-4-5-20250929" >Anthropic claude-sonnet-4-5-20250929</option>
+                    <option value="gpt-5.5" >gpt-5.5</option>
                     <option value="openai_gpt-5.2" >OpenAI gpt-5.2</option>
                 </select>
             </div>
@@ -428,10 +430,10 @@
 
     <script>
         // All models data embedded
-        const allModelsData = {"anthropic_claude-opus-4-5-20251101": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 4, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 3, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "sourcery": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 3}, "core": {"tp": 1, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 2}, "core": {"tp": 2, "fp": 6, "fn": 2}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "bugbot": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 7, "fn": 0}, "core": {"tp": 2, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 5}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 5}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 4, "fp": 13, "fn": 2}, "core": {"tp": 4, "fp": 13, "fn": 2}, "all": {"tp": 4, "fp": 13, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 3}, "core": {"tp": 3, "fp": 5, "fn": 3}, "all": {"tp": 3, "fp": 5, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 6}, "core": {"tp": 0, "fp": 2, "fn": 6}, "all": {"tp": 0, "fp": 2, "fn": 6}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 0}, "core": {"tp": 4, "fp": 1, "fn": 0}, "all": {"tp": 4, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 8, "fn": 1}, "core": {"tp": 3, "fp": 8, "fn": 1}, "all": {"tp": 3, "fp": 8, "fn": 1}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 3, "fp": 7, "fn": 1}, "core": {"tp": 3, "fp": 7, "fn": 1}, "all": {"tp": 3, "fp": 7, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 2}, "all": {"tp": 1, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 13, "fn": 1}, "core": {"tp": 2, "fp": 13, "fn": 1}, "all": {"tp": 2, "fp": 13, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 1}}, "claude": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 10, "fn": 2}, "core": {"tp": 0, "fp": 10, "fn": 2}, "all": {"tp": 0, "fp": 10, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "coderabbit": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 4}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 5}, "core": {"tp": 2, "fp": 0, "fn": 5}, "all": {"tp": 2, "fp": 0, "fn": 5}}, "codeant-v2": {"strict": {"tp": 4, "fp": 5, "fn": 3}, "core": {"tp": 4, "fp": 5, "fn": 3}, "all": {"tp": 4, "fp": 5, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 8, "fn": 4}, "core": {"tp": 3, "fp": 8, "fn": 4}, "all": {"tp": 3, "fp": 8, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 5}, "core": {"tp": 2, "fp": 2, "fn": 5}, "all": {"tp": 2, "fp": 2, "fn": 5}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 3}, "core": {"tp": 4, "fp": 2, "fn": 3}, "all": {"tp": 4, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "copilot-v2": {"strict": {"tp": 6, "fp": 5, "fn": 1}, "core": {"tp": 6, "fp": 5, "fn": 1}, "all": {"tp": 6, "fp": 5, "fn": 1}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "bugbot": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 6}, "core": {"tp": 1, "fp": 2, "fn": 6}, "all": {"tp": 1, "fp": 2, "fn": 6}}, "claude-code": {"strict": {"tp": 2, "fp": 4, "fn": 5}, "core": {"tp": 2, "fp": 4, "fn": 5}, "all": {"tp": 2, "fp": 4, "fn": 5}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 6}, "core": {"tp": 1, "fp": 1, "fn": 6}, "all": {"tp": 1, "fp": 1, "fn": 6}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 7}, "core": {"tp": 0, "fp": 0, "fn": 7}, "all": {"tp": 0, "fp": 0, "fn": 7}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 6}, "core": {"tp": 1, "fp": 1, "fn": 6}, "all": {"tp": 1, "fp": 1, "fn": 6}}, "kodus-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "macroscope": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 4, "fn": 4}, "core": {"tp": 3, "fp": 4, "fn": 4}, "all": {"tp": 3, "fp": 4, "fn": 4}}, "sourcery": {"strict": {"tp": 5, "fp": 7, "fn": 2}, "core": {"tp": 5, "fp": 7, "fn": 2}, "all": {"tp": 5, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 8, "fn": 3}, "core": {"tp": 0, "fp": 8, "fn": 3}, "all": {"tp": 1, "fp": 8, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 5, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 4, "fp": 6, "fn": 2}, "core": {"tp": 4, "fp": 6, "fn": 2}, "all": {"tp": 4, "fp": 6, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 9, "fn": 3}, "core": {"tp": 3, "fp": 9, "fn": 3}, "all": {"tp": 3, "fp": 9, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 1, "fp": 6, "fn": 5}, "core": {"tp": 1, "fp": 6, "fn": 5}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "sourcery": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 6}, "core": {"tp": 2, "fp": 2, "fn": 6}, "all": {"tp": 2, "fp": 2, "fn": 6}}, "codeant-v2": {"strict": {"tp": 3, "fp": 16, "fn": 5}, "core": {"tp": 3, "fp": 16, "fn": 5}, "all": {"tp": 3, "fp": 16, "fn": 5}}, "coderabbit": {"strict": {"tp": 6, "fp": 19, "fn": 2}, "core": {"tp": 6, "fp": 19, "fn": 2}, "all": {"tp": 6, "fp": 19, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 5, "fp": 4, "fn": 3}, "core": {"tp": 5, "fp": 4, "fn": 3}, "all": {"tp": 5, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 5, "fp": 0, "fn": 3}, "core": {"tp": 5, "fp": 0, "fn": 3}, "all": {"tp": 5, "fp": 0, "fn": 3}}, "copilot-v2": {"strict": {"tp": 6, "fp": 2, "fn": 2}, "core": {"tp": 6, "fp": 2, "fn": 2}, "all": {"tp": 6, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 5, "fp": 0, "fn": 3}, "core": {"tp": 5, "fp": 0, "fn": 3}, "all": {"tp": 5, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 4}, "core": {"tp": 4, "fp": 3, "fn": 4}, "all": {"tp": 4, "fp": 3, "fn": 4}}, "claude-code": {"strict": {"tp": 5, "fp": 0, "fn": 3}, "core": {"tp": 5, "fp": 0, "fn": 3}, "all": {"tp": 5, "fp": 0, "fn": 3}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kodus-v2": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 6}, "core": {"tp": 2, "fp": 4, "fn": 6}, "all": {"tp": 2, "fp": 4, "fn": 6}}, "qodo-v2": {"strict": {"tp": 6, "fp": 5, "fn": 2}, "core": {"tp": 6, "fp": 5, "fn": 2}, "all": {"tp": 6, "fp": 5, "fn": 2}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "codeant-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 9, "fn": 2}, "core": {"tp": 2, "fp": 9, "fn": 2}, "all": {"tp": 2, "fp": 9, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "copilot-v2": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 6, "fp": 0, "fn": 3}, "core": {"tp": 6, "fp": 0, "fn": 3}, "all": {"tp": 6, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 7, "fp": 4, "fn": 2}, "core": {"tp": 7, "fp": 4, "fn": 2}, "all": {"tp": 7, "fp": 4, "fn": 2}}, "coderabbit": {"strict": {"tp": 9, "fp": 10, "fn": 0}, "core": {"tp": 9, "fp": 10, "fn": 0}, "all": {"tp": 9, "fp": 10, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 7, "fp": 2, "fn": 2}, "core": {"tp": 7, "fp": 2, "fn": 2}, "all": {"tp": 7, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 2, "fn": 4}, "core": {"tp": 5, "fp": 2, "fn": 4}, "all": {"tp": 5, "fp": 2, "fn": 4}}, "gemini-v2": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 6, "fp": 2, "fn": 3}, "core": {"tp": 6, "fp": 2, "fn": 3}, "all": {"tp": 6, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 6, "fp": 2, "fn": 3}, "core": {"tp": 6, "fp": 2, "fn": 3}, "all": {"tp": 6, "fp": 2, "fn": 3}}, "augment": {"strict": {"tp": 9, "fp": 1, "fn": 0}, "core": {"tp": 9, "fp": 1, "fn": 0}, "all": {"tp": 9, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 6, "fp": 4, "fn": 3}, "core": {"tp": 6, "fp": 4, "fn": 3}, "all": {"tp": 6, "fp": 4, "fn": 3}}, "claude": {"strict": {"tp": 6, "fp": 0, "fn": 3}, "core": {"tp": 6, "fp": 0, "fn": 3}, "all": {"tp": 6, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 7, "fp": 0, "fn": 2}, "core": {"tp": 7, "fp": 0, "fn": 2}, "all": {"tp": 7, "fp": 0, "fn": 2}}, "cubic-v2": {"strict": {"tp": 4, "fp": 0, "fn": 5}, "core": {"tp": 4, "fp": 0, "fn": 5}, "all": {"tp": 4, "fp": 0, "fn": 5}}, "devin": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 8}, "core": {"tp": 1, "fp": 0, "fn": 8}, "all": {"tp": 1, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 9}, "core": {"tp": 0, "fp": 0, "fn": 9}, "all": {"tp": 0, "fp": 0, "fn": 9}}, "kodus-v2": {"strict": {"tp": 4, "fp": 3, "fn": 5}, "core": {"tp": 4, "fp": 3, "fn": 5}, "all": {"tp": 4, "fp": 3, "fn": 5}}, "macroscope": {"strict": {"tp": 7, "fp": 6, "fn": 2}, "core": {"tp": 7, "fp": 6, "fn": 2}, "all": {"tp": 7, "fp": 6, "fn": 2}}, "qodo-v2": {"strict": {"tp": 8, "fp": 0, "fn": 1}, "core": {"tp": 8, "fp": 0, "fn": 1}, "all": {"tp": 8, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 9, "fn": 1}, "core": {"tp": 2, "fp": 9, "fn": 1}, "all": {"tp": 2, "fp": 9, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "coderabbit": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "copilot-v2": {"strict": {"tp": 5, "fp": 3, "fn": 1}, "core": {"tp": 5, "fp": 3, "fn": 1}, "all": {"tp": 5, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 4, "fp": 2, "fn": 2}, "core": {"tp": 4, "fp": 2, "fn": 2}, "all": {"tp": 4, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "qodo-v2": {"strict": {"tp": 5, "fp": 2, "fn": 1}, "core": {"tp": 5, "fp": 2, "fn": 1}, "all": {"tp": 5, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 4, "fp": 4, "fn": 0}, "all": {"tp": 5, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 3}, "core": {"tp": 2, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 5, "fp": 7, "fn": 0}, "core": {"tp": 5, "fp": 7, "fn": 0}, "all": {"tp": 5, "fp": 7, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 4, "fn": 3}, "core": {"tp": 2, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 4}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 5, "fp": 6, "fn": 0}, "core": {"tp": 5, "fp": 6, "fn": 0}, "all": {"tp": 5, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 0, "fn": 1}, "core": {"tp": 4, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 5, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "devin": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 2, "fp": 6, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 10, "fn": 0}, "core": {"tp": 1, "fp": 10, "fn": 0}, "all": {"tp": 2, "fp": 10, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}], "tools": ["augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"augment": {"strict": {"precision": 56.5, "recall": 65.5, "f1": 60.7, "f2": 63.5, "tp": 91, "fp": 70, "fn": 48, "num_prs": 50}, "core": {"precision": 59.5, "recall": 65.2, "f1": 62.2, "f2": 64.0, "tp": 103, "fp": 70, "fn": 55, "num_prs": 50}, "all": {"precision": 60.7, "recall": 62.4, "f1": 61.5, "f2": 62.1, "tp": 108, "fp": 70, "fn": 65, "num_prs": 50}}, "cubic-v2": {"strict": {"precision": 58.3, "recall": 65.5, "f1": 61.7, "f2": 63.9, "tp": 91, "fp": 65, "fn": 48, "num_prs": 50}, "core": {"precision": 61.5, "recall": 65.8, "f1": 63.6, "f2": 64.9, "tp": 104, "fp": 65, "fn": 54, "num_prs": 50}, "all": {"precision": 62.4, "recall": 62.4, "f1": 62.4, "f2": 62.4, "tp": 108, "fp": 65, "fn": 65, "num_prs": 50}}, "copilot-v2": {"strict": {"precision": 35.6, "recall": 66.9, "f1": 46.5, "f2": 56.9, "tp": 93, "fp": 168, "fn": 46, "num_prs": 50}, "core": {"precision": 38.5, "recall": 66.5, "f1": 48.7, "f2": 58.0, "tp": 105, "fp": 168, "fn": 53, "num_prs": 50}, "all": {"precision": 40.6, "recall": 66.5, "f1": 50.4, "f2": 59.0, "tp": 115, "fp": 168, "fn": 58, "num_prs": 50}}, "greptile-v4-1": {"strict": {"precision": 49.0, "recall": 54.0, "f1": 51.4, "f2": 52.9, "tp": 75, "fp": 78, "fn": 64, "num_prs": 50}, "core": {"precision": 50.9, "recall": 51.3, "f1": 51.1, "f2": 51.2, "tp": 81, "fp": 78, "fn": 77, "num_prs": 50}, "all": {"precision": 53.3, "recall": 51.4, "f1": 52.4, "f2": 51.8, "tp": 89, "fp": 78, "fn": 84, "num_prs": 50}}, "sourcery": {"strict": {"precision": 38.4, "recall": 56.1, "f1": 45.6, "f2": 51.4, "tp": 78, "fp": 125, "fn": 61, "num_prs": 50}, "core": {"precision": 41.0, "recall": 55.1, "f1": 47.0, "f2": 51.5, "tp": 87, "fp": 125, "fn": 71, "num_prs": 50}, "all": {"precision": 43.2, "recall": 54.9, "f1": 48.3, "f2": 52.1, "tp": 95, "fp": 125, "fn": 78, "num_prs": 50}}, "qodo-v2": {"strict": {"precision": 53.0, "recall": 64.0, "f1": 58.0, "f2": 61.5, "tp": 89, "fp": 79, "fn": 50, "num_prs": 50}, "core": {"precision": 55.4, "recall": 62.0, "f1": 58.5, "f2": 60.6, "tp": 98, "fp": 79, "fn": 60, "num_prs": 50}, "all": {"precision": 56.4, "recall": 59.0, "f1": 57.6, "f2": 58.4, "tp": 102, "fp": 79, "fn": 71, "num_prs": 50}}, "gitlab": {"strict": {"precision": 51.0, "recall": 54.0, "f1": 52.4, "f2": 53.3, "tp": 75, "fp": 72, "fn": 64, "num_prs": 50}, "core": {"precision": 53.5, "recall": 52.5, "f1": 53.0, "f2": 52.7, "tp": 83, "fp": 72, "fn": 75, "num_prs": 50}, "all": {"precision": 55.8, "recall": 52.6, "f1": 54.2, "f2": 53.2, "tp": 91, "fp": 72, "fn": 82, "num_prs": 50}}, "codeant-v2": {"strict": {"precision": 39.0, "recall": 44.6, "f1": 41.6, "f2": 43.4, "tp": 62, "fp": 97, "fn": 77, "num_prs": 50}, "core": {"precision": 40.1, "recall": 41.1, "f1": 40.6, "f2": 40.9, "tp": 65, "fp": 97, "fn": 93, "num_prs": 50}, "all": {"precision": 41.2, "recall": 39.3, "f1": 40.2, "f2": 39.7, "tp": 68, "fp": 97, "fn": 105, "num_prs": 50}}, "bugbot": {"strict": {"precision": 54.8, "recall": 48.9, "f1": 51.7, "f2": 50.0, "tp": 68, "fp": 56, "fn": 71, "num_prs": 50}, "core": {"precision": 56.9, "recall": 46.8, "f1": 51.4, "f2": 48.6, "tp": 74, "fp": 56, "fn": 84, "num_prs": 50}, "all": {"precision": 58.2, "recall": 45.1, "f1": 50.8, "f2": 47.2, "tp": 78, "fp": 56, "fn": 95, "num_prs": 50}}, "claude-code": {"strict": {"precision": 44.3, "recall": 50.4, "f1": 47.1, "f2": 49.0, "tp": 70, "fp": 88, "fn": 69, "num_prs": 50}, "core": {"precision": 46.3, "recall": 48.1, "f1": 47.2, "f2": 47.7, "tp": 76, "fp": 88, "fn": 82, "num_prs": 50}, "all": {"precision": 48.8, "recall": 48.6, "f1": 48.7, "f2": 48.6, "tp": 84, "fp": 88, "fn": 89, "num_prs": 50}}, "devin": {"strict": {"precision": 67.4, "recall": 43.2, "f1": 52.6, "f2": 46.5, "tp": 60, "fp": 29, "fn": 79, "num_prs": 50}, "core": {"precision": 69.1, "recall": 41.1, "f1": 51.6, "f2": 44.8, "tp": 65, "fp": 29, "fn": 93, "num_prs": 50}, "all": {"precision": 70.1, "recall": 39.3, "f1": 50.4, "f2": 43.1, "tp": 68, "fp": 29, "fn": 105, "num_prs": 50}}, "macroscope": {"strict": {"precision": 60.2, "recall": 53.2, "f1": 56.5, "f2": 54.5, "tp": 74, "fp": 49, "fn": 65, "num_prs": 50}, "core": {"precision": 61.1, "recall": 48.7, "f1": 54.2, "f2": 50.8, "tp": 77, "fp": 49, "fn": 81, "num_prs": 50}, "all": {"precision": 62.3, "recall": 46.8, "f1": 53.5, "f2": 49.3, "tp": 81, "fp": 49, "fn": 92, "num_prs": 50}}, "claude": {"strict": {"precision": 41.0, "recall": 39.6, "f1": 40.3, "f2": 39.9, "tp": 55, "fp": 79, "fn": 84, "num_prs": 50}, "core": {"precision": 43.2, "recall": 38.0, "f1": 40.4, "f2": 38.9, "tp": 60, "fp": 79, "fn": 98, "num_prs": 50}, "all": {"precision": 45.5, "recall": 38.2, "f1": 41.5, "f2": 39.4, "tp": 66, "fp": 79, "fn": 107, "num_prs": 50}}, "gemini-v2": {"strict": {"precision": 43.2, "recall": 57.6, "f1": 49.4, "f2": 54.0, "tp": 80, "fp": 105, "fn": 59, "num_prs": 50}, "core": {"precision": 45.3, "recall": 55.1, "f1": 49.7, "f2": 52.8, "tp": 87, "fp": 105, "fn": 71, "num_prs": 50}, "all": {"precision": 47.5, "recall": 54.9, "f1": 50.9, "f2": 53.3, "tp": 95, "fp": 105, "fn": 78, "num_prs": 50}}, "coderabbit": {"strict": {"precision": 31.0, "recall": 62.6, "f1": 41.4, "f2": 52.0, "tp": 87, "fp": 194, "fn": 52, "num_prs": 50}, "core": {"precision": 32.6, "recall": 59.5, "f1": 42.2, "f2": 51.1, "tp": 94, "fp": 194, "fn": 64, "num_prs": 50}, "all": {"precision": 34.7, "recall": 59.5, "f1": 43.8, "f2": 52.1, "tp": 103, "fp": 194, "fn": 70, "num_prs": 50}}, "baz": {"strict": {"precision": 60.6, "recall": 41.0, "f1": 48.9, "f2": 43.8, "tp": 57, "fp": 37, "fn": 82, "num_prs": 50}, "core": {"precision": 61.5, "recall": 37.3, "f1": 46.5, "f2": 40.5, "tp": 59, "fp": 37, "fn": 99, "num_prs": 50}, "all": {"precision": 62.6, "recall": 35.8, "f1": 45.6, "f2": 39.2, "tp": 62, "fp": 37, "fn": 111, "num_prs": 50}}, "kg": {"strict": {"precision": 53.3, "recall": 17.3, "f1": 26.1, "f2": 20.0, "tp": 24, "fp": 21, "fn": 115, "num_prs": 50}, "core": {"precision": 54.3, "recall": 15.8, "f1": 24.5, "f2": 18.4, "tp": 25, "fp": 21, "fn": 133, "num_prs": 50}, "all": {"precision": 55.3, "recall": 15.0, "f1": 23.6, "f2": 17.6, "tp": 26, "fp": 21, "fn": 147, "num_prs": 50}}, "qodo-extended-v2": {"strict": {"precision": 64.8, "recall": 66.2, "f1": 65.5, "f2": 65.9, "tp": 92, "fp": 50, "fn": 47, "num_prs": 50}, "core": {"precision": 67.1, "recall": 64.6, "f1": 65.8, "f2": 65.1, "tp": 102, "fp": 50, "fn": 56, "num_prs": 50}, "all": {"precision": 67.9, "recall": 61.3, "f1": 64.4, "f2": 62.5, "tp": 106, "fp": 50, "fn": 67, "num_prs": 50}}, "kodus-v2": {"strict": {"precision": 57.1, "recall": 40.3, "f1": 47.3, "f2": 42.8, "tp": 56, "fp": 42, "fn": 83, "num_prs": 50}, "core": {"precision": 59.2, "recall": 38.6, "f1": 46.7, "f2": 41.5, "tp": 61, "fp": 42, "fn": 97, "num_prs": 50}, "all": {"precision": 59.6, "recall": 35.8, "f1": 44.8, "f2": 38.9, "tp": 62, "fp": 42, "fn": 111, "num_prs": 50}}, "graphite": {"strict": {"precision": 100.0, "recall": 7.9, "f1": 14.7, "f2": 9.7, "tp": 11, "fp": 0, "fn": 128, "num_prs": 50}, "core": {"precision": 100.0, "recall": 7.6, "f1": 14.1, "f2": 9.3, "tp": 12, "fp": 0, "fn": 146, "num_prs": 50}, "all": {"precision": 100.0, "recall": 6.9, "f1": 13.0, "f2": 8.5, "tp": 12, "fp": 0, "fn": 161, "num_prs": 50}}}}, "anthropic_claude-sonnet-4-5-20250929": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 4, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "sourcery": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 6, "fn": 0}, "core": {"tp": 4, "fp": 6, "fn": 0}, "all": {"tp": 4, "fp": 6, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 1, "fp": 6, "fn": 4}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 8, "fn": 0}, "core": {"tp": 2, "fp": 8, "fn": 0}, "all": {"tp": 2, "fp": 8, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 5}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 4}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 1, "fp": 5, "fn": 2}, "core": {"tp": 3, "fp": 5, "fn": 2}, "all": {"tp": 3, "fp": 5, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 6}, "core": {"tp": 0, "fp": 2, "fn": 6}, "all": {"tp": 0, "fp": 2, "fn": 6}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 4, "fp": 13, "fn": 2}, "core": {"tp": 4, "fp": 13, "fn": 2}, "all": {"tp": 4, "fp": 13, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 5}, "core": {"tp": 1, "fp": 3, "fn": 5}, "all": {"tp": 1, "fp": 3, "fn": 5}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 3}, "core": {"tp": 3, "fp": 5, "fn": 3}, "all": {"tp": 3, "fp": 5, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 7, "fn": 2}, "core": {"tp": 2, "fp": 7, "fn": 2}, "all": {"tp": 2, "fp": 7, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 2}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 1, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 0}, "core": {"tp": 4, "fp": 1, "fn": 0}, "all": {"tp": 4, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 10, "fn": 1}, "core": {"tp": 3, "fp": 10, "fn": 1}, "all": {"tp": 3, "fp": 10, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 7, "fn": 2}, "core": {"tp": 1, "fp": 7, "fn": 2}, "all": {"tp": 1, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 0, "fp": 5, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 8, "fn": 2}, "core": {"tp": 1, "fp": 8, "fn": 2}, "all": {"tp": 1, "fp": 8, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 13, "fn": 1}, "core": {"tp": 2, "fp": 13, "fn": 1}, "all": {"tp": 2, "fp": 13, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 2, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "claude-code": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 3, "fp": 4, "fn": 0}}, "codeant-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 6, "fn": 0}, "core": {"tp": 3, "fp": 6, "fn": 0}, "all": {"tp": 3, "fp": 6, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 7, "fn": 2}, "core": {"tp": 1, "fp": 7, "fn": 2}, "all": {"tp": 1, "fp": 7, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 6, "fn": 0}, "core": {"tp": 3, "fp": 6, "fn": 0}, "all": {"tp": 3, "fp": 6, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 12, "fn": 2}, "core": {"tp": 0, "fp": 12, "fn": 2}, "all": {"tp": 0, "fp": 12, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "coderabbit": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 2, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 4, "fn": 4}, "core": {"tp": 0, "fp": 4, "fn": 5}, "all": {"tp": 0, "fp": 4, "fn": 5}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 4}, "core": {"tp": 3, "fp": 2, "fn": 4}, "all": {"tp": 3, "fp": 2, "fn": 4}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 5}, "core": {"tp": 2, "fp": 0, "fn": 5}, "all": {"tp": 2, "fp": 0, "fn": 5}}, "bugbot": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 6}, "core": {"tp": 1, "fp": 2, "fn": 6}, "all": {"tp": 1, "fp": 2, "fn": 6}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 6}, "core": {"tp": 1, "fp": 5, "fn": 6}, "all": {"tp": 1, "fp": 5, "fn": 6}}, "codeant-v2": {"strict": {"tp": 2, "fp": 7, "fn": 5}, "core": {"tp": 2, "fp": 7, "fn": 5}, "all": {"tp": 2, "fp": 7, "fn": 5}}, "coderabbit": {"strict": {"tp": 4, "fp": 10, "fn": 3}, "core": {"tp": 4, "fp": 10, "fn": 3}, "all": {"tp": 4, "fp": 10, "fn": 3}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 6}, "core": {"tp": 1, "fp": 1, "fn": 6}, "all": {"tp": 1, "fp": 1, "fn": 6}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 7}, "core": {"tp": 0, "fp": 0, "fn": 7}, "all": {"tp": 0, "fp": 0, "fn": 7}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 6}, "core": {"tp": 1, "fp": 0, "fn": 6}, "all": {"tp": 1, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 5, "fp": 3, "fn": 2}, "core": {"tp": 5, "fp": 3, "fn": 2}, "all": {"tp": 5, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 4, "fn": 4}, "core": {"tp": 3, "fp": 4, "fn": 4}, "all": {"tp": 3, "fp": 4, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 5, "fn": 5}, "core": {"tp": 2, "fp": 5, "fn": 5}, "all": {"tp": 2, "fp": 5, "fn": 5}}, "sourcery": {"strict": {"tp": 4, "fp": 7, "fn": 3}, "core": {"tp": 4, "fp": 7, "fn": 3}, "all": {"tp": 4, "fp": 7, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 5}, "core": {"tp": 2, "fp": 2, "fn": 5}, "all": {"tp": 2, "fp": 2, "fn": 5}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 3}, "core": {"tp": 4, "fp": 2, "fn": 3}, "all": {"tp": 4, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "copilot-v2": {"strict": {"tp": 6, "fp": 5, "fn": 1}, "core": {"tp": 6, "fp": 5, "fn": 1}, "all": {"tp": 6, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 9, "fn": 3}, "core": {"tp": 0, "fp": 9, "fn": 3}, "all": {"tp": 0, "fp": 9, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "codeant-v2": {"strict": {"tp": 4, "fp": 9, "fn": 2}, "core": {"tp": 4, "fp": 9, "fn": 2}, "all": {"tp": 4, "fp": 9, "fn": 2}}, "coderabbit": {"strict": {"tp": 4, "fp": 6, "fn": 2}, "core": {"tp": 4, "fp": 6, "fn": 2}, "all": {"tp": 4, "fp": 6, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 1, "fp": 6, "fn": 5}, "core": {"tp": 1, "fp": 6, "fn": 5}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 3, "fn": 5}, "core": {"tp": 1, "fp": 3, "fn": 5}, "all": {"tp": 1, "fp": 3, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 3, "fp": 14, "fn": 3}, "core": {"tp": 3, "fp": 14, "fn": 3}, "all": {"tp": 3, "fp": 14, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 2}, "all": {"tp": 1, "fp": 6, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "baz": {"strict": {"tp": 2, "fp": 2, "fn": 6}, "core": {"tp": 2, "fp": 2, "fn": 6}, "all": {"tp": 2, "fp": 2, "fn": 6}}, "bugbot": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}, "claude": {"strict": {"tp": 6, "fp": 2, "fn": 2}, "core": {"tp": 6, "fp": 2, "fn": 2}, "all": {"tp": 6, "fp": 2, "fn": 2}}, "claude-code": {"strict": {"tp": 4, "fp": 0, "fn": 4}, "core": {"tp": 4, "fp": 0, "fn": 4}, "all": {"tp": 4, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 3, "fp": 16, "fn": 5}, "core": {"tp": 3, "fp": 16, "fn": 5}, "all": {"tp": 3, "fp": 16, "fn": 5}}, "coderabbit": {"strict": {"tp": 5, "fp": 11, "fn": 3}, "core": {"tp": 5, "fp": 11, "fn": 3}, "all": {"tp": 5, "fp": 11, "fn": 3}}, "cubic-v2": {"strict": {"tp": 1, "fp": 4, "fn": 7}, "core": {"tp": 1, "fp": 4, "fn": 7}, "all": {"tp": 1, "fp": 4, "fn": 7}}, "devin": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kodus-v2": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "macroscope": {"strict": {"tp": 2, "fp": 5, "fn": 6}, "core": {"tp": 2, "fp": 5, "fn": 6}, "all": {"tp": 2, "fp": 5, "fn": 6}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "qodo-v2": {"strict": {"tp": 6, "fp": 3, "fn": 2}, "core": {"tp": 6, "fp": 3, "fn": 2}, "all": {"tp": 6, "fp": 3, "fn": 2}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 1, "fn": 3}, "core": {"tp": 5, "fp": 1, "fn": 3}, "all": {"tp": 5, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 4, "fp": 6, "fn": 4}, "core": {"tp": 4, "fp": 6, "fn": 4}, "all": {"tp": 4, "fp": 6, "fn": 4}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 5, "fp": 5, "fn": 3}, "core": {"tp": 5, "fp": 5, "fn": 3}, "all": {"tp": 5, "fp": 5, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 5, "fn": 4}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 0, "fp": 5, "fn": 4}}, "codeant-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kg": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 8, "fp": 2, "fn": 1}, "core": {"tp": 8, "fp": 2, "fn": 1}, "all": {"tp": 8, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "bugbot": {"strict": {"tp": 6, "fp": 1, "fn": 3}, "core": {"tp": 6, "fp": 1, "fn": 3}, "all": {"tp": 6, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 4, "fp": 0, "fn": 5}, "core": {"tp": 4, "fp": 0, "fn": 5}, "all": {"tp": 4, "fp": 0, "fn": 5}}, "codeant-v2": {"strict": {"tp": 4, "fp": 4, "fn": 5}, "core": {"tp": 4, "fp": 4, "fn": 5}, "all": {"tp": 4, "fp": 4, "fn": 5}}, "coderabbit": {"strict": {"tp": 9, "fp": 9, "fn": 0}, "core": {"tp": 9, "fp": 9, "fn": 0}, "all": {"tp": 9, "fp": 9, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 6}, "core": {"tp": 3, "fp": 0, "fn": 6}, "all": {"tp": 3, "fp": 0, "fn": 6}}, "devin": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 8}, "core": {"tp": 1, "fp": 0, "fn": 8}, "all": {"tp": 1, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 9}, "core": {"tp": 0, "fp": 0, "fn": 9}, "all": {"tp": 0, "fp": 0, "fn": 9}}, "kodus-v2": {"strict": {"tp": 4, "fp": 3, "fn": 5}, "core": {"tp": 4, "fp": 3, "fn": 5}, "all": {"tp": 4, "fp": 3, "fn": 5}}, "macroscope": {"strict": {"tp": 5, "fp": 5, "fn": 4}, "core": {"tp": 5, "fp": 5, "fn": 4}, "all": {"tp": 5, "fp": 5, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 7, "fp": 2, "fn": 2}, "core": {"tp": 7, "fp": 2, "fn": 2}, "all": {"tp": 7, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 8, "fp": 3, "fn": 1}, "core": {"tp": 8, "fp": 3, "fn": 1}, "all": {"tp": 8, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 3, "fn": 4}, "core": {"tp": 5, "fp": 3, "fn": 4}, "all": {"tp": 5, "fp": 3, "fn": 4}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 5}, "core": {"tp": 4, "fp": 2, "fn": 5}, "all": {"tp": 4, "fp": 2, "fn": 5}}, "gitlab": {"strict": {"tp": 5, "fp": 2, "fn": 4}, "core": {"tp": 5, "fp": 2, "fn": 4}, "all": {"tp": 5, "fp": 2, "fn": 4}}, "copilot-v2": {"strict": {"tp": 6, "fp": 1, "fn": 3}, "core": {"tp": 6, "fp": 1, "fn": 3}, "all": {"tp": 6, "fp": 1, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "codeant-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "coderabbit": {"strict": {"tp": 4, "fp": 4, "fn": 2}, "core": {"tp": 4, "fp": 4, "fn": 2}, "all": {"tp": 4, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 4, "fp": 2, "fn": 2}, "core": {"tp": 4, "fp": 2, "fn": 2}, "all": {"tp": 4, "fp": 2, "fn": 2}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "copilot-v2": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 4, "fp": 4, "fn": 0}, "all": {"tp": 5, "fp": 4, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 2, "fp": 4, "fn": 3}, "core": {"tp": 2, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 4}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 5, "fn": 2}, "core": {"tp": 3, "fp": 5, "fn": 2}, "all": {"tp": 3, "fp": 5, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 7, "fn": 4}, "core": {"tp": 1, "fp": 7, "fn": 4}, "all": {"tp": 1, "fp": 7, "fn": 5}}, "coderabbit": {"strict": {"tp": 5, "fp": 5, "fn": 0}, "core": {"tp": 5, "fp": 5, "fn": 0}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 3, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 4, "fn": 4}, "core": {"tp": 1, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 5}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 5, "fn": 3}, "core": {"tp": 2, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "gemini-v2": {"strict": {"tp": 1, "fp": 6, "fn": 4}, "core": {"tp": 1, "fp": 6, "fn": 4}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "gitlab": {"strict": {"tp": 3, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "copilot-v2": {"strict": {"tp": 4, "fp": 7, "fn": 1}, "core": {"tp": 4, "fp": 7, "fn": 1}, "all": {"tp": 4, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 9, "fn": 0}, "core": {"tp": 1, "fp": 9, "fn": 0}, "all": {"tp": 2, "fp": 9, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}], "tools": ["augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"augment": {"strict": {"precision": 49.4, "recall": 61.9, "f1": 55.0, "f2": 58.9, "tp": 86, "fp": 88, "fn": 53, "num_prs": 50}, "core": {"precision": 52.4, "recall": 61.4, "f1": 56.6, "f2": 59.4, "tp": 97, "fp": 88, "fn": 61, "num_prs": 50}, "all": {"precision": 53.7, "recall": 59.0, "f1": 56.2, "f2": 57.8, "tp": 102, "fp": 88, "fn": 71, "num_prs": 50}}, "cubic-v2": {"strict": {"precision": 52.9, "recall": 59.7, "f1": 56.1, "f2": 58.2, "tp": 83, "fp": 74, "fn": 56, "num_prs": 50}, "core": {"precision": 56.5, "recall": 60.8, "f1": 58.5, "f2": 59.9, "tp": 96, "fp": 74, "fn": 62, "num_prs": 50}, "all": {"precision": 57.2, "recall": 57.2, "f1": 57.2, "f2": 57.2, "tp": 99, "fp": 74, "fn": 74, "num_prs": 50}}, "copilot-v2": {"strict": {"precision": 31.4, "recall": 61.9, "f1": 41.6, "f2": 51.8, "tp": 86, "fp": 188, "fn": 53, "num_prs": 50}, "core": {"precision": 34.3, "recall": 62.0, "f1": 44.1, "f2": 53.4, "tp": 98, "fp": 188, "fn": 60, "num_prs": 50}, "all": {"precision": 36.1, "recall": 61.3, "f1": 45.4, "f2": 53.8, "tp": 106, "fp": 188, "fn": 67, "num_prs": 50}}, "greptile-v4-1": {"strict": {"precision": 44.9, "recall": 50.4, "f1": 47.5, "f2": 49.2, "tp": 70, "fp": 86, "fn": 69, "num_prs": 50}, "core": {"precision": 46.9, "recall": 48.1, "f1": 47.5, "f2": 47.9, "tp": 76, "fp": 86, "fn": 82, "num_prs": 50}, "all": {"precision": 49.1, "recall": 48.0, "f1": 48.5, "f2": 48.2, "tp": 83, "fp": 86, "fn": 90, "num_prs": 50}}, "sourcery": {"strict": {"precision": 34.2, "recall": 48.9, "f1": 40.2, "f2": 45.0, "tp": 68, "fp": 131, "fn": 71, "num_prs": 50}, "core": {"precision": 37.0, "recall": 48.7, "f1": 42.1, "f2": 45.8, "tp": 77, "fp": 131, "fn": 81, "num_prs": 50}, "all": {"precision": 38.8, "recall": 48.0, "f1": 42.9, "f2": 45.8, "tp": 83, "fp": 131, "fn": 90, "num_prs": 50}}, "qodo-v2": {"strict": {"precision": 45.1, "recall": 59.0, "f1": 51.1, "f2": 55.6, "tp": 82, "fp": 100, "fn": 57, "num_prs": 50}, "core": {"precision": 47.1, "recall": 56.3, "f1": 51.3, "f2": 54.2, "tp": 89, "fp": 100, "fn": 69, "num_prs": 50}, "all": {"precision": 47.9, "recall": 53.2, "f1": 50.4, "f2": 52.0, "tp": 92, "fp": 100, "fn": 81, "num_prs": 50}}, "gitlab": {"strict": {"precision": 44.7, "recall": 48.9, "f1": 46.7, "f2": 48.0, "tp": 68, "fp": 84, "fn": 71, "num_prs": 50}, "core": {"precision": 47.5, "recall": 48.1, "f1": 47.8, "f2": 48.0, "tp": 76, "fp": 84, "fn": 82, "num_prs": 50}, "all": {"precision": 49.7, "recall": 48.0, "f1": 48.8, "f2": 48.3, "tp": 83, "fp": 84, "fn": 90, "num_prs": 50}}, "codeant-v2": {"strict": {"precision": 31.1, "recall": 37.4, "f1": 34.0, "f2": 36.0, "tp": 52, "fp": 115, "fn": 87, "num_prs": 50}, "core": {"precision": 32.4, "recall": 34.8, "f1": 33.5, "f2": 34.3, "tp": 55, "fp": 115, "fn": 103, "num_prs": 50}, "all": {"precision": 33.1, "recall": 32.9, "f1": 33.0, "f2": 33.0, "tp": 57, "fp": 115, "fn": 116, "num_prs": 50}}, "bugbot": {"strict": {"precision": 52.0, "recall": 46.0, "f1": 48.9, "f2": 47.1, "tp": 64, "fp": 59, "fn": 75, "num_prs": 50}, "core": {"precision": 54.3, "recall": 44.3, "f1": 48.8, "f2": 46.0, "tp": 70, "fp": 59, "fn": 88, "num_prs": 50}, "all": {"precision": 55.3, "recall": 42.2, "f1": 47.9, "f2": 44.3, "tp": 73, "fp": 59, "fn": 100, "num_prs": 50}}, "claude-code": {"strict": {"precision": 34.3, "recall": 42.4, "f1": 37.9, "f2": 40.5, "tp": 59, "fp": 113, "fn": 80, "num_prs": 50}, "core": {"precision": 36.5, "recall": 41.1, "f1": 38.7, "f2": 40.1, "tp": 65, "fp": 113, "fn": 93, "num_prs": 50}, "all": {"precision": 38.3, "recall": 40.5, "f1": 39.3, "f2": 40.0, "tp": 70, "fp": 113, "fn": 103, "num_prs": 50}}, "devin": {"strict": {"precision": 66.0, "recall": 44.6, "f1": 53.2, "f2": 47.7, "tp": 62, "fp": 32, "fn": 77, "num_prs": 50}, "core": {"precision": 67.3, "recall": 41.8, "f1": 51.6, "f2": 45.2, "tp": 66, "fp": 32, "fn": 92, "num_prs": 50}, "all": {"precision": 68.3, "recall": 39.9, "f1": 50.4, "f2": 43.5, "tp": 69, "fp": 32, "fn": 104, "num_prs": 50}}, "macroscope": {"strict": {"precision": 50.4, "recall": 45.3, "f1": 47.7, "f2": 46.3, "tp": 63, "fp": 62, "fn": 76, "num_prs": 50}, "core": {"precision": 51.6, "recall": 41.8, "f1": 46.2, "f2": 43.4, "tp": 66, "fp": 62, "fn": 92, "num_prs": 50}, "all": {"precision": 53.0, "recall": 40.5, "f1": 45.9, "f2": 42.5, "tp": 70, "fp": 62, "fn": 103, "num_prs": 50}}, "claude": {"strict": {"precision": 36.1, "recall": 37.4, "f1": 36.7, "f2": 37.1, "tp": 52, "fp": 92, "fn": 87, "num_prs": 50}, "core": {"precision": 38.3, "recall": 36.1, "f1": 37.1, "f2": 36.5, "tp": 57, "fp": 92, "fn": 101, "num_prs": 50}, "all": {"precision": 40.6, "recall": 36.4, "f1": 38.4, "f2": 37.2, "tp": 63, "fp": 92, "fn": 110, "num_prs": 50}}, "gemini-v2": {"strict": {"precision": 36.7, "recall": 51.8, "f1": 43.0, "f2": 47.9, "tp": 72, "fp": 124, "fn": 67, "num_prs": 50}, "core": {"precision": 39.2, "recall": 50.6, "f1": 44.2, "f2": 47.8, "tp": 80, "fp": 124, "fn": 78, "num_prs": 50}, "all": {"precision": 41.2, "recall": 50.3, "f1": 45.3, "f2": 48.2, "tp": 87, "fp": 124, "fn": 86, "num_prs": 50}}, "coderabbit": {"strict": {"precision": 31.8, "recall": 60.4, "f1": 41.7, "f2": 51.2, "tp": 84, "fp": 180, "fn": 55, "num_prs": 50}, "core": {"precision": 33.8, "recall": 58.2, "f1": 42.8, "f2": 50.9, "tp": 92, "fp": 180, "fn": 66, "num_prs": 50}, "all": {"precision": 35.3, "recall": 56.6, "f1": 43.5, "f2": 50.5, "tp": 98, "fp": 180, "fn": 75, "num_prs": 50}}, "baz": {"strict": {"precision": 54.6, "recall": 38.1, "f1": 44.9, "f2": 40.6, "tp": 53, "fp": 44, "fn": 86, "num_prs": 50}, "core": {"precision": 55.6, "recall": 34.8, "f1": 42.8, "f2": 37.6, "tp": 55, "fp": 44, "fn": 103, "num_prs": 50}, "all": {"precision": 56.9, "recall": 33.5, "f1": 42.2, "f2": 36.5, "tp": 58, "fp": 44, "fn": 115, "num_prs": 50}}, "kg": {"strict": {"precision": 48.8, "recall": 15.1, "f1": 23.1, "f2": 17.5, "tp": 21, "fp": 22, "fn": 118, "num_prs": 50}, "core": {"precision": 50.0, "recall": 13.9, "f1": 21.8, "f2": 16.3, "tp": 22, "fp": 22, "fn": 136, "num_prs": 50}, "all": {"precision": 51.1, "recall": 13.3, "f1": 21.1, "f2": 15.6, "tp": 23, "fp": 22, "fn": 150, "num_prs": 50}}, "qodo-extended-v2": {"strict": {"precision": 53.6, "recall": 58.3, "f1": 55.9, "f2": 57.3, "tp": 81, "fp": 70, "fn": 58, "num_prs": 50}, "core": {"precision": 56.2, "recall": 57.0, "f1": 56.6, "f2": 56.8, "tp": 90, "fp": 70, "fn": 68, "num_prs": 50}, "all": {"precision": 56.8, "recall": 53.2, "f1": 54.9, "f2": 53.9, "tp": 92, "fp": 70, "fn": 81, "num_prs": 50}}, "kodus-v2": {"strict": {"precision": 55.2, "recall": 38.1, "f1": 45.1, "f2": 40.6, "tp": 53, "fp": 43, "fn": 86, "num_prs": 50}, "core": {"precision": 57.4, "recall": 36.7, "f1": 44.8, "f2": 39.6, "tp": 58, "fp": 43, "fn": 100, "num_prs": 50}, "all": {"precision": 57.8, "recall": 34.1, "f1": 42.9, "f2": 37.2, "tp": 59, "fp": 43, "fn": 114, "num_prs": 50}}, "graphite": {"strict": {"precision": 100.0, "recall": 7.9, "f1": 14.7, "f2": 9.7, "tp": 11, "fp": 0, "fn": 128, "num_prs": 50}, "core": {"precision": 100.0, "recall": 7.6, "f1": 14.1, "f2": 9.3, "tp": 12, "fp": 0, "fn": 146, "num_prs": 50}, "all": {"precision": 100.0, "recall": 6.9, "f1": 13.0, "f2": 8.5, "tp": 12, "fp": 0, "fn": 161, "num_prs": 50}}}}, "openai_gpt-5.2": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 4, "fp": 6, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 3, "fp": 5, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 3, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "sourcery": {"strict": {"tp": 0, "fp": 13, "fn": 1}, "core": {"tp": 0, "fp": 13, "fn": 3}, "all": {"tp": 2, "fp": 13, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 2}, "all": {"tp": 2, "fp": 5, "fn": 2}}, "coderabbit": {"strict": {"tp": 3, "fp": 7, "fn": 0}, "core": {"tp": 3, "fp": 7, "fn": 1}, "all": {"tp": 3, "fp": 7, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 3}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 8, "fn": 2}, "core": {"tp": 0, "fp": 8, "fn": 2}, "all": {"tp": 0, "fp": 8, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 8, "fn": 1}, "core": {"tp": 1, "fp": 8, "fn": 1}, "all": {"tp": 1, "fp": 8, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 7, "fn": 2}, "core": {"tp": 0, "fp": 7, "fn": 2}, "all": {"tp": 0, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 3}, "core": {"tp": 1, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 7, "fn": 3}, "core": {"tp": 1, "fp": 7, "fn": 3}, "all": {"tp": 1, "fp": 7, "fn": 4}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 8, "fn": 0}, "core": {"tp": 2, "fp": 8, "fn": 0}, "all": {"tp": 2, "fp": 8, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 4, "fp": 4, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 3, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 4}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 5}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 4, "fp": 13, "fn": 2}, "core": {"tp": 4, "fp": 13, "fn": 2}, "all": {"tp": 4, "fp": 13, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 3}, "core": {"tp": 3, "fp": 5, "fn": 3}, "all": {"tp": 3, "fp": 5, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 6}, "core": {"tp": 0, "fp": 2, "fn": 6}, "all": {"tp": 0, "fp": 2, "fn": 6}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 6, "fn": 2}, "core": {"tp": 2, "fp": 6, "fn": 2}, "all": {"tp": 2, "fp": 6, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 7, "fn": 1}, "core": {"tp": 1, "fp": 7, "fn": 1}, "all": {"tp": 1, "fp": 7, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 8, "fn": 0}, "core": {"tp": 0, "fp": 8, "fn": 1}, "all": {"tp": 0, "fp": 8, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 11, "fn": 0}, "core": {"tp": 0, "fp": 11, "fn": 1}, "all": {"tp": 0, "fp": 11, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 0}, "core": {"tp": 4, "fp": 1, "fn": 0}, "all": {"tp": 4, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 10, "fn": 1}, "core": {"tp": 3, "fp": 10, "fn": 1}, "all": {"tp": 3, "fp": 10, "fn": 1}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 3, "fp": 7, "fn": 1}, "core": {"tp": 3, "fp": 7, "fn": 1}, "all": {"tp": 3, "fp": 7, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 6, "fn": 3}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 9, "fn": 2}, "core": {"tp": 1, "fp": 9, "fn": 2}, "all": {"tp": 1, "fp": 9, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 13, "fn": 1}, "core": {"tp": 2, "fp": 13, "fn": 1}, "all": {"tp": 2, "fp": 13, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 8, "fn": 2}, "core": {"tp": 1, "fp": 8, "fn": 2}, "all": {"tp": 1, "fp": 8, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 2, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 3, "fp": 4, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 3, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 1}, "core": {"tp": 1, "fp": 7, "fn": 1}, "all": {"tp": 1, "fp": 7, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 9, "fn": 2}, "core": {"tp": 0, "fp": 9, "fn": 2}, "all": {"tp": 0, "fp": 9, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "coderabbit": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 4, "fp": 5, "fn": 1}, "all": {"tp": 4, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 4, "fn": 4}, "core": {"tp": 0, "fp": 4, "fn": 5}, "all": {"tp": 0, "fp": 4, "fn": 5}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 2}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 5}, "core": {"tp": 2, "fp": 1, "fn": 5}, "all": {"tp": 2, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 7, "fn": 5}, "core": {"tp": 2, "fp": 7, "fn": 5}, "all": {"tp": 2, "fp": 7, "fn": 5}}, "coderabbit": {"strict": {"tp": 5, "fp": 14, "fn": 2}, "core": {"tp": 5, "fp": 14, "fn": 2}, "all": {"tp": 5, "fp": 14, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 6, "fn": 3}, "core": {"tp": 4, "fp": 6, "fn": 3}, "all": {"tp": 4, "fp": 6, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 3}, "core": {"tp": 4, "fp": 2, "fn": 3}, "all": {"tp": 4, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "copilot-v2": {"strict": {"tp": 6, "fp": 5, "fn": 1}, "core": {"tp": 6, "fp": 5, "fn": 1}, "all": {"tp": 6, "fp": 5, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 1, "fn": 3}, "core": {"tp": 4, "fp": 1, "fn": 3}, "all": {"tp": 4, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 6}, "core": {"tp": 1, "fp": 2, "fn": 6}, "all": {"tp": 1, "fp": 2, "fn": 6}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 6}, "core": {"tp": 1, "fp": 5, "fn": 6}, "all": {"tp": 1, "fp": 5, "fn": 6}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 5}, "core": {"tp": 2, "fp": 0, "fn": 5}, "all": {"tp": 2, "fp": 0, "fn": 5}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 7}, "core": {"tp": 0, "fp": 0, "fn": 7}, "all": {"tp": 0, "fp": 0, "fn": 7}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 6}, "core": {"tp": 1, "fp": 0, "fn": 6}, "all": {"tp": 1, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "macroscope": {"strict": {"tp": 4, "fp": 3, "fn": 3}, "core": {"tp": 4, "fp": 3, "fn": 3}, "all": {"tp": 4, "fp": 3, "fn": 3}}, "qodo-v2": {"strict": {"tp": 4, "fp": 6, "fn": 3}, "core": {"tp": 4, "fp": 6, "fn": 3}, "all": {"tp": 4, "fp": 6, "fn": 3}}, "sourcery": {"strict": {"tp": 5, "fp": 7, "fn": 2}, "core": {"tp": 5, "fp": 7, "fn": 2}, "all": {"tp": 5, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 9, "fn": 3}, "core": {"tp": 0, "fp": 9, "fn": 3}, "all": {"tp": 0, "fp": 9, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 5, "fp": 3, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 5, "fp": 7, "fn": 1}, "core": {"tp": 5, "fp": 7, "fn": 1}, "all": {"tp": 5, "fp": 7, "fn": 1}}, "coderabbit": {"strict": {"tp": 4, "fp": 6, "fn": 2}, "core": {"tp": 4, "fp": 6, "fn": 2}, "all": {"tp": 4, "fp": 6, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 8, "fn": 3}, "core": {"tp": 3, "fp": 8, "fn": 3}, "all": {"tp": 3, "fp": 8, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 4, "fp": 4, "fn": 2}, "core": {"tp": 4, "fp": 4, "fn": 2}, "all": {"tp": 4, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 1, "fp": 6, "fn": 5}, "core": {"tp": 1, "fp": 6, "fn": 5}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 6}, "core": {"tp": 2, "fp": 2, "fn": 6}, "all": {"tp": 2, "fp": 2, "fn": 6}}, "codeant-v2": {"strict": {"tp": 1, "fp": 17, "fn": 7}, "core": {"tp": 1, "fp": 17, "fn": 7}, "all": {"tp": 1, "fp": 17, "fn": 7}}, "coderabbit": {"strict": {"tp": 4, "fp": 23, "fn": 4}, "core": {"tp": 4, "fp": 23, "fn": 4}, "all": {"tp": 4, "fp": 23, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 3, "fn": 5}, "core": {"tp": 3, "fp": 3, "fn": 5}, "all": {"tp": 3, "fp": 3, "fn": 5}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 4, "fp": 5, "fn": 4}, "core": {"tp": 4, "fp": 5, "fn": 4}, "all": {"tp": 4, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 5, "fp": 6, "fn": 3}, "core": {"tp": 5, "fp": 6, "fn": 3}, "all": {"tp": 5, "fp": 6, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "bugbot": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}, "claude": {"strict": {"tp": 3, "fp": 3, "fn": 5}, "core": {"tp": 3, "fp": 3, "fn": 5}, "all": {"tp": 3, "fp": 3, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 7}, "core": {"tp": 1, "fp": 3, "fn": 7}, "all": {"tp": 1, "fp": 3, "fn": 7}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kodus-v2": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 5, "fn": 7}, "core": {"tp": 1, "fp": 5, "fn": 7}, "all": {"tp": 1, "fp": 5, "fn": 7}}, "qodo-v2": {"strict": {"tp": 5, "fp": 13, "fn": 3}, "core": {"tp": 5, "fp": 13, "fn": 3}, "all": {"tp": 5, "fp": 13, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "codeant-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "copilot-v2": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 5, "fn": 0}, "core": {"tp": 2, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 4, "fp": 1, "fn": 5}, "core": {"tp": 4, "fp": 1, "fn": 5}, "all": {"tp": 4, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 4, "fp": 5, "fn": 5}, "core": {"tp": 4, "fp": 5, "fn": 5}, "all": {"tp": 4, "fp": 5, "fn": 5}}, "coderabbit": {"strict": {"tp": 8, "fp": 13, "fn": 1}, "core": {"tp": 8, "fp": 13, "fn": 1}, "all": {"tp": 8, "fp": 13, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 7, "fp": 2, "fn": 2}, "core": {"tp": 7, "fp": 2, "fn": 2}, "all": {"tp": 7, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 4, "fn": 4}, "core": {"tp": 5, "fp": 4, "fn": 4}, "all": {"tp": 5, "fp": 4, "fn": 4}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 5}, "core": {"tp": 4, "fp": 2, "fn": 5}, "all": {"tp": 4, "fp": 2, "fn": 5}}, "gitlab": {"strict": {"tp": 5, "fp": 5, "fn": 4}, "core": {"tp": 5, "fp": 5, "fn": 4}, "all": {"tp": 5, "fp": 5, "fn": 4}}, "copilot-v2": {"strict": {"tp": 7, "fp": 3, "fn": 2}, "core": {"tp": 7, "fp": 3, "fn": 2}, "all": {"tp": 7, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 8, "fp": 5, "fn": 1}, "core": {"tp": 8, "fp": 5, "fn": 1}, "all": {"tp": 8, "fp": 5, "fn": 1}}, "bugbot": {"strict": {"tp": 6, "fp": 2, "fn": 3}, "core": {"tp": 6, "fp": 2, "fn": 3}, "all": {"tp": 6, "fp": 2, "fn": 3}}, "claude": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 4, "fp": 0, "fn": 5}, "core": {"tp": 4, "fp": 0, "fn": 5}, "all": {"tp": 4, "fp": 0, "fn": 5}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 6}, "core": {"tp": 3, "fp": 0, "fn": 6}, "all": {"tp": 3, "fp": 0, "fn": 6}}, "devin": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 8}, "core": {"tp": 1, "fp": 1, "fn": 8}, "all": {"tp": 1, "fp": 1, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 9}, "core": {"tp": 0, "fp": 0, "fn": 9}, "all": {"tp": 0, "fp": 0, "fn": 9}}, "kodus-v2": {"strict": {"tp": 4, "fp": 3, "fn": 5}, "core": {"tp": 4, "fp": 3, "fn": 5}, "all": {"tp": 4, "fp": 3, "fn": 5}}, "macroscope": {"strict": {"tp": 5, "fp": 5, "fn": 4}, "core": {"tp": 5, "fp": 5, "fn": 4}, "all": {"tp": 5, "fp": 5, "fn": 4}}, "qodo-v2": {"strict": {"tp": 9, "fp": 2, "fn": 0}, "core": {"tp": 9, "fp": 2, "fn": 0}, "all": {"tp": 9, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 12, "fn": 1}, "core": {"tp": 2, "fp": 12, "fn": 1}, "all": {"tp": 2, "fp": 12, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "coderabbit": {"strict": {"tp": 4, "fp": 7, "fn": 2}, "core": {"tp": 4, "fp": 7, "fn": 2}, "all": {"tp": 4, "fp": 7, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "copilot-v2": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "qodo-v2": {"strict": {"tp": 5, "fp": 3, "fn": 1}, "core": {"tp": 5, "fp": 3, "fn": 1}, "all": {"tp": 5, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 1}, "core": {"tp": 1, "fp": 7, "fn": 3}, "all": {"tp": 2, "fp": 7, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 4, "fp": 3, "fn": 0}, "all": {"tp": 5, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 10, "fn": 2}, "core": {"tp": 1, "fp": 10, "fn": 3}, "all": {"tp": 2, "fp": 10, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 4}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "qodo-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 2}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 6, "fn": 3}, "core": {"tp": 2, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 4}}, "coderabbit": {"strict": {"tp": 5, "fp": 15, "fn": 0}, "core": {"tp": 5, "fp": 15, "fn": 0}, "all": {"tp": 5, "fp": 15, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "gemini-v2": {"strict": {"tp": 1, "fp": 6, "fn": 4}, "core": {"tp": 1, "fp": 6, "fn": 4}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 4, "fp": 8, "fn": 1}, "core": {"tp": 4, "fp": 8, "fn": 1}, "all": {"tp": 4, "fp": 8, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 3, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "claude-code": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 1, "fp": 7, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}], "tools": ["augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"augment": {"strict": {"precision": 45.9, "recall": 59.7, "f1": 51.9, "f2": 56.3, "tp": 83, "fp": 98, "fn": 56, "num_prs": 50}, "core": {"precision": 49.5, "recall": 60.8, "f1": 54.5, "f2": 58.1, "tp": 96, "fp": 98, "fn": 62, "num_prs": 50}, "all": {"precision": 50.8, "recall": 58.4, "f1": 54.3, "f2": 56.7, "tp": 101, "fp": 98, "fn": 72, "num_prs": 50}}, "cubic-v2": {"strict": {"precision": 53.2, "recall": 59.7, "f1": 56.3, "f2": 58.3, "tp": 83, "fp": 73, "fn": 56, "num_prs": 50}, "core": {"precision": 56.8, "recall": 60.8, "f1": 58.7, "f2": 59.9, "tp": 96, "fp": 73, "fn": 62, "num_prs": 50}, "all": {"precision": 57.8, "recall": 57.8, "f1": 57.8, "f2": 57.8, "tp": 100, "fp": 73, "fn": 73, "num_prs": 50}}, "copilot-v2": {"strict": {"precision": 30.4, "recall": 63.3, "f1": 41.1, "f2": 52.1, "tp": 88, "fp": 201, "fn": 51, "num_prs": 50}, "core": {"precision": 33.2, "recall": 63.3, "f1": 43.6, "f2": 53.6, "tp": 100, "fp": 201, "fn": 58, "num_prs": 50}, "all": {"precision": 35.2, "recall": 63.0, "f1": 45.1, "f2": 54.4, "tp": 109, "fp": 201, "fn": 64, "num_prs": 50}}, "greptile-v4-1": {"strict": {"precision": 40.2, "recall": 50.4, "f1": 44.7, "f2": 47.9, "tp": 70, "fp": 104, "fn": 69, "num_prs": 50}, "core": {"precision": 42.2, "recall": 48.1, "f1": 45.0, "f2": 46.8, "tp": 76, "fp": 104, "fn": 82, "num_prs": 50}, "all": {"precision": 44.7, "recall": 48.6, "f1": 46.5, "f2": 47.7, "tp": 84, "fp": 104, "fn": 89, "num_prs": 50}}, "sourcery": {"strict": {"precision": 33.3, "recall": 50.4, "f1": 40.1, "f2": 45.7, "tp": 70, "fp": 140, "fn": 69, "num_prs": 50}, "core": {"precision": 36.1, "recall": 50.0, "f1": 41.9, "f2": 46.4, "tp": 79, "fp": 140, "fn": 79, "num_prs": 50}, "all": {"precision": 38.1, "recall": 49.7, "f1": 43.1, "f2": 46.8, "tp": 86, "fp": 140, "fn": 87, "num_prs": 50}}, "qodo-v2": {"strict": {"precision": 44.1, "recall": 64.7, "f1": 52.5, "f2": 59.2, "tp": 90, "fp": 114, "fn": 49, "num_prs": 50}, "core": {"precision": 46.5, "recall": 62.7, "f1": 53.4, "f2": 58.6, "tp": 99, "fp": 114, "fn": 59, "num_prs": 50}, "all": {"precision": 47.5, "recall": 59.5, "f1": 52.8, "f2": 56.7, "tp": 103, "fp": 114, "fn": 70, "num_prs": 50}}, "gitlab": {"strict": {"precision": 42.2, "recall": 48.9, "f1": 45.3, "f2": 47.4, "tp": 68, "fp": 93, "fn": 71, "num_prs": 50}, "core": {"precision": 45.0, "recall": 48.1, "f1": 46.5, "f2": 47.4, "tp": 76, "fp": 93, "fn": 82, "num_prs": 50}, "all": {"precision": 46.9, "recall": 47.4, "f1": 47.1, "f2": 47.3, "tp": 82, "fp": 93, "fn": 91, "num_prs": 50}}, "codeant-v2": {"strict": {"precision": 29.3, "recall": 38.1, "f1": 33.1, "f2": 36.0, "tp": 53, "fp": 128, "fn": 86, "num_prs": 50}, "core": {"precision": 30.4, "recall": 35.4, "f1": 32.7, "f2": 34.3, "tp": 56, "fp": 128, "fn": 102, "num_prs": 50}, "all": {"precision": 31.2, "recall": 33.5, "f1": 32.3, "f2": 33.0, "tp": 58, "fp": 128, "fn": 115, "num_prs": 50}}, "bugbot": {"strict": {"precision": 49.6, "recall": 47.5, "f1": 48.5, "f2": 47.9, "tp": 66, "fp": 67, "fn": 73, "num_prs": 50}, "core": {"precision": 51.8, "recall": 45.6, "f1": 48.5, "f2": 46.7, "tp": 72, "fp": 67, "fn": 86, "num_prs": 50}, "all": {"precision": 52.8, "recall": 43.4, "f1": 47.6, "f2": 45.0, "tp": 75, "fp": 67, "fn": 98, "num_prs": 50}}, "claude-code": {"strict": {"precision": 38.2, "recall": 43.2, "f1": 40.5, "f2": 42.1, "tp": 60, "fp": 97, "fn": 79, "num_prs": 50}, "core": {"precision": 40.1, "recall": 41.1, "f1": 40.6, "f2": 40.9, "tp": 65, "fp": 97, "fn": 93, "num_prs": 50}, "all": {"precision": 42.3, "recall": 41.0, "f1": 41.6, "f2": 41.3, "tp": 71, "fp": 97, "fn": 102, "num_prs": 50}}, "devin": {"strict": {"precision": 56.6, "recall": 43.2, "f1": 49.0, "f2": 45.3, "tp": 60, "fp": 46, "fn": 79, "num_prs": 50}, "core": {"precision": 58.6, "recall": 41.1, "f1": 48.3, "f2": 43.7, "tp": 65, "fp": 46, "fn": 93, "num_prs": 50}, "all": {"precision": 59.3, "recall": 38.7, "f1": 46.9, "f2": 41.6, "tp": 67, "fp": 46, "fn": 106, "num_prs": 50}}, "macroscope": {"strict": {"precision": 47.7, "recall": 44.6, "f1": 46.1, "f2": 45.2, "tp": 62, "fp": 68, "fn": 77, "num_prs": 50}, "core": {"precision": 48.9, "recall": 41.1, "f1": 44.7, "f2": 42.5, "tp": 65, "fp": 68, "fn": 93, "num_prs": 50}, "all": {"precision": 50.0, "recall": 39.3, "f1": 44.0, "f2": 41.1, "tp": 68, "fp": 68, "fn": 105, "num_prs": 50}}, "claude": {"strict": {"precision": 36.7, "recall": 38.8, "f1": 37.8, "f2": 38.4, "tp": 54, "fp": 93, "fn": 85, "num_prs": 50}, "core": {"precision": 38.8, "recall": 37.3, "f1": 38.1, "f2": 37.6, "tp": 59, "fp": 93, "fn": 99, "num_prs": 50}, "all": {"precision": 41.1, "recall": 37.6, "f1": 39.3, "f2": 38.2, "tp": 65, "fp": 93, "fn": 108, "num_prs": 50}}, "gemini-v2": {"strict": {"precision": 36.2, "recall": 54.0, "f1": 43.4, "f2": 49.1, "tp": 75, "fp": 132, "fn": 64, "num_prs": 50}, "core": {"precision": 38.0, "recall": 51.3, "f1": 43.7, "f2": 47.9, "tp": 81, "fp": 132, "fn": 77, "num_prs": 50}, "all": {"precision": 40.3, "recall": 51.4, "f1": 45.2, "f2": 48.7, "tp": 89, "fp": 132, "fn": 84, "num_prs": 50}}, "coderabbit": {"strict": {"precision": 26.8, "recall": 61.9, "f1": 37.4, "f2": 49.0, "tp": 86, "fp": 235, "fn": 53, "num_prs": 50}, "core": {"precision": 28.6, "recall": 59.5, "f1": 38.6, "f2": 48.9, "tp": 94, "fp": 235, "fn": 64, "num_prs": 50}, "all": {"precision": 30.7, "recall": 60.1, "f1": 40.6, "f2": 50.4, "tp": 104, "fp": 235, "fn": 69, "num_prs": 50}}, "baz": {"strict": {"precision": 43.9, "recall": 36.0, "f1": 39.5, "f2": 37.3, "tp": 50, "fp": 64, "fn": 89, "num_prs": 50}, "core": {"precision": 44.8, "recall": 32.9, "f1": 38.0, "f2": 34.8, "tp": 52, "fp": 64, "fn": 106, "num_prs": 50}, "all": {"precision": 46.2, "recall": 31.8, "f1": 37.7, "f2": 33.9, "tp": 55, "fp": 64, "fn": 118, "num_prs": 50}}, "kg": {"strict": {"precision": 53.5, "recall": 16.5, "f1": 25.3, "f2": 19.2, "tp": 23, "fp": 20, "fn": 116, "num_prs": 50}, "core": {"precision": 54.5, "recall": 15.2, "f1": 23.8, "f2": 17.8, "tp": 24, "fp": 20, "fn": 134, "num_prs": 50}, "all": {"precision": 55.6, "recall": 14.5, "f1": 22.9, "f2": 17.0, "tp": 25, "fp": 20, "fn": 148, "num_prs": 50}}, "qodo-extended-v2": {"strict": {"precision": 55.9, "recall": 61.2, "f1": 58.4, "f2": 60.0, "tp": 85, "fp": 67, "fn": 54, "num_prs": 50}, "core": {"precision": 58.4, "recall": 59.5, "f1": 58.9, "f2": 59.3, "tp": 94, "fp": 67, "fn": 64, "num_prs": 50}, "all": {"precision": 58.9, "recall": 55.5, "f1": 57.1, "f2": 56.1, "tp": 96, "fp": 67, "fn": 77, "num_prs": 50}}, "kodus-v2": {"strict": {"precision": 50.5, "recall": 36.0, "f1": 42.0, "f2": 38.2, "tp": 50, "fp": 49, "fn": 89, "num_prs": 50}, "core": {"precision": 52.4, "recall": 34.2, "f1": 41.4, "f2": 36.7, "tp": 54, "fp": 49, "fn": 104, "num_prs": 50}, "all": {"precision": 53.8, "recall": 32.9, "f1": 40.9, "f2": 35.7, "tp": 57, "fp": 49, "fn": 116, "num_prs": 50}}, "graphite": {"strict": {"precision": 71.4, "recall": 7.2, "f1": 13.1, "f2": 8.8, "tp": 10, "fp": 4, "fn": 129, "num_prs": 50}, "core": {"precision": 73.3, "recall": 7.0, "f1": 12.7, "f2": 8.5, "tp": 11, "fp": 4, "fn": 147, "num_prs": 50}, "all": {"precision": 73.3, "recall": 6.4, "f1": 11.7, "f2": 7.8, "tp": 11, "fp": 4, "fn": 162, "num_prs": 50}}}}};
-        const toolDisplayNames = {"graphite": "Graphite", "qodo": "Qodo", "gemini": "Gemini", "claude": "Claude Code", "augment": "Augment", "bugbot": "Cursor Bugbot", "coderabbit": "CodeRabbit", "propel": "Propel", "copilot": "GitHub Copilot", "baz": "Baz", "greptile": "Greptile", "kg": "KG", "entelligence": "Entelligence", "cubic-v2": "Cubic v2", "sourcery": "Sourcery", "mesa": "Mesa", "codeant-v2": "CodeAnt", "claude-code": "Claude Code (CLI)", "devin": "Devin", "kodus-v2": "Kodus", "greptile-v4-1": "Greptile v4", "gemini-v2": "Gemini", "gitlab": "GitLab Duo Code Review", "qodo-v2": "Qodo v2", "qodo-extended-v2": "Qodo Extended", "macroscope": "Macroscope"};
-        const toolColors = {"graphite": "#6366f1", "qodo": "#8b5cf6", "gemini": "#06b6d4", "claude": "#f59e0b", "augment": "#10b981", "bugbot": "#3b82f6", "coderabbit": "#ec4899", "propel": "#14b8a6", "propel-v2": "#0d9488", "copilot": "#6b7280", "baz": "#f97316", "greptile": "#22c55e", "kg": "#a855f7", "entelligence": "#0ea5e9", "cubic-dev": "#d946ef", "sourcery": "#84cc16", "mesa": "#f43f5e", "codeant": "#e11d48", "codeant-v2": "#fb7185", "claude-code": "#d97706", "devin": "#7c3aed", "kodus-v2": "#059669", "greptile-v4": "#16a34a", "qodo-v2": "#7c3aed", "qodo-extended-v2": "#6d28d9", "macroscope": "#0891b2", "cubic-v2": "#c026d3"};
-        const predefinedFilters = [{"id": "high_precision", "label": "Highest Precision", "filters": {}, "sort": "precision", "description": "Tools ranked by precision - fewer false positives, more reliable findings", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "graphite", "best_score": 100.0}, {"id": "tool_kg_language_ruby", "label": "Best for Ruby (Precision)", "filters": {"language": ["Ruby"]}, "description": "Ruby codebases with Rails patterns.", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "kg", "best_score": 100.0, "sort": "precision"}, {"id": "tool_sourcery_difficulty_moderate_context_file", "label": "Moderate Bugs + File Context (Recall)", "filters": {"difficulty": ["moderate"], "context": ["file"]}, "description": "Requires careful reading. Requires full file understanding.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "sourcery", "best_score": 90.0, "sort": "recall"}, {"id": "tool_coderabbit_language_typescript_domain_scheduling", "label": "Typescript + Scheduling (Recall)", "filters": {"language": ["TypeScript"], "domain": ["scheduling"]}, "description": "TypeScript codebases with frontend patterns. Task scheduling.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "coderabbit", "best_score": 88.9, "sort": "recall"}, {"id": "go_small", "label": "Best for Small Go PRs", "filters": {"language": ["Go"], "pr_size": ["small"]}, "description": "Go codebases with concurrency patterns. Small PRs with 1-2 files, easier to review thoroughly", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 88.2}, {"id": "complexity_complex", "label": "Best for Complex Code", "filters": {"complexity": ["complex"]}, "description": "Deep logic and dependencies.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 81.1}, {"id": "domain_ui", "label": "Best for Ui", "filters": {"domain": ["UI"]}, "description": "User interface and frontend.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "bugbot", "best_score": 78.3}, {"id": "risk_critical", "label": "Best for Critical Risk", "filters": {"risk": ["critical"]}, "description": "Critical security or data corruption risk.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-v2", "best_score": 75.9}, {"id": "tool_codeant-v2_pr_size_small_change_type_performance", "label": "Small PRs + Performance Optimization (Precision)", "filters": {"pr_size": ["small"], "change_type": ["performance"]}, "description": "Small PRs with 1-2 files, easier to review thoroughly Performance optimization changes", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "codeant-v2", "best_score": 75.0, "sort": "precision"}, {"id": "tool_gitlab_change_type_bug_fix_context_cross_file", "label": "Bug Fixes + Cross-File", "filters": {"change_type": ["bug_fix"], "context": ["cross_file"]}, "description": "Bug fixes and issue resolution Spans multiple files.", "best_model": "openai_gpt-5.2", "best_tool": "gitlab", "best_score": 74.3}, {"id": "tool_baz_language_java_domain_authentication", "label": "Java + Authentication", "filters": {"language": ["Java"], "domain": ["authentication"]}, "description": "Java codebases with OOP patterns. Auth and access control.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "baz", "best_score": 74.1}, {"id": "python_medium", "label": "Best for Medium Python PRs", "filters": {"language": ["Python"], "pr_size": ["medium"]}, "description": "Python codebases with dynamic typing. Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 72.7}, {"id": "domain_concurrency", "label": "Best for Concurrency", "filters": {"domain": ["concurrency"]}, "description": "Threading and async operations.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "macroscope", "best_score": 69.6}, {"id": "tool_claude_change_type_bug_fix", "label": "Best for Bug Fixes", "filters": {"change_type": ["bug_fix"]}, "description": "Bug fixes and issue resolution", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "claude", "best_score": 68.3}, {"id": "tool_greptile-v4-1_language_ruby_pr_size_medium", "label": "Ruby + Medium PRs", "filters": {"language": ["Ruby"], "pr_size": ["medium"]}, "description": "Ruby codebases with Rails patterns. Medium PRs with 3-5 files, typical feature development", "best_model": "openai_gpt-5.2", "best_tool": "greptile-v4-1", "best_score": 66.7}, {"id": "high_recall", "label": "Highest Recall", "filters": {}, "sort": "recall", "description": "Tools ranked by recall - catches more issues, may have more noise", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "copilot-v2", "best_score": 66.5}, {"id": "tool_kodus-v2_language_ruby_change_type_feature", "label": "Ruby + Features", "filters": {"language": ["Ruby"], "change_type": ["feature"]}, "description": "Ruby codebases with Rails patterns. New feature implementation", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "kodus-v2", "best_score": 61.2}, {"id": "difficulty_moderate", "label": "Best for Moderate Bugs", "filters": {"difficulty": ["moderate"]}, "description": "Requires careful reading.", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "devin", "best_score": 57.9}, {"id": "tool_gemini-v2_language_ruby_difficulty_subtle", "label": "Ruby + Subtle Bugs", "filters": {"language": ["Ruby"], "difficulty": ["subtle"]}, "description": "Ruby codebases with Rails patterns. Non-obvious, needs domain knowledge.", "best_model": "openai_gpt-5.2", "best_tool": "gemini-v2", "best_score": 56.4}, {"id": "language_go", "label": "Best for Go", "filters": {"language": ["Go"]}, "description": "Go codebases with concurrency patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 84.2}, {"id": "change_type_performance", "label": "Best for Performance Optimization", "filters": {"change_type": ["performance"]}, "description": "Performance optimization changes", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 82.9}, {"id": "complex_subtle", "label": "Complex & Subtle", "filters": {"complexity": ["complex"], "difficulty": ["subtle", "very_subtle"]}, "description": "Deep logic and dependencies. Non-obvious, needs domain knowledge.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 81.1}, {"id": "pr_size_small", "label": "Best for Small PRs", "filters": {"pr_size": ["small"]}, "description": "Small PRs with 1-2 files, easier to review thoroughly", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 79.6}, {"id": "domain_caching", "label": "Best for Caching", "filters": {"domain": ["caching"]}, "description": "Cache and memoization.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 78.6}, {"id": "pr_size_large", "label": "Best for Large PRs", "filters": {"pr_size": ["large"]}, "description": "Large PRs with 6+ files, complex changes requiring careful review", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 74.5}, {"id": "change_type_bug_fix", "label": "Best for Bug Fixes", "filters": {"change_type": ["bug_fix"]}, "description": "Bug fixes and issue resolution", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 73.2}, {"id": "language_python", "label": "Best for Python", "filters": {"language": ["Python"]}, "description": "Python codebases with dynamic typing.", "best_model": "openai_gpt-5.2", "best_tool": "cubic-v2", "best_score": 72.5}, {"id": "high_risk_auth", "label": "High Risk Auth", "filters": {"risk": ["high", "critical"], "domain": ["authentication"]}, "description": "Significant impact, potential data loss. Auth and access control.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 71.9}, {"id": "context_file", "label": "Best for File Context", "filters": {"context": ["file"]}, "description": "Requires full file understanding.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 71.6}, {"id": "risk_high", "label": "Best for High Risk", "filters": {"risk": ["high"]}, "description": "Significant impact, potential data loss.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 71.3}, {"id": "language_java", "label": "Best for Java", "filters": {"language": ["Java"]}, "description": "Java codebases with OOP patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 70.8}, {"id": "domain_authentication", "label": "Best for Authentication", "filters": {"domain": ["authentication"]}, "description": "Auth and access control.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 69.2}, {"id": "ruby_medium", "label": "Best for Medium Ruby PRs", "filters": {"language": ["Ruby"], "pr_size": ["medium"]}, "description": "Ruby codebases with Rails patterns. Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 68.8}, {"id": "java_medium", "label": "Best for Medium Java PRs", "filters": {"language": ["Java"], "pr_size": ["medium"]}, "description": "Java codebases with OOP patterns. Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 68.6}, {"id": "domain_scheduling", "label": "Best for Scheduling", "filters": {"domain": ["scheduling"]}, "description": "Task scheduling.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 68.3}, {"id": "language_typescript", "label": "Best for Typescript", "filters": {"language": ["TypeScript"]}, "description": "TypeScript codebases with frontend patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 68.2}, {"id": "context_cross_file", "label": "Best for Cross-File", "filters": {"context": ["cross_file"]}, "description": "Spans multiple files.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 67.5}, {"id": "pr_size_medium", "label": "Best for Medium PRs", "filters": {"pr_size": ["medium"]}, "description": "Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 67.1}, {"id": "concern_reliability", "label": "Best for Reliability", "filters": {"concern": ["reliability"]}, "description": "Error handling and system stability", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-v2", "best_score": 66.7}, {"id": "difficulty_subtle", "label": "Best for Subtle Bugs", "filters": {"difficulty": ["subtle"]}, "description": "Non-obvious, needs domain knowledge.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 66.4}, {"id": "concern_correctness", "label": "Best for Correctness", "filters": {"concern": ["correctness"]}, "description": "Logical correctness and expected behavior", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 66.4}, {"id": "high_f1", "label": "Highest Score", "filters": {}, "sort": "f1", "description": "Tools ranked by F-beta score (default F2, recall-weighted)", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 65.8}, {"id": "concern_security", "label": "Best for Security", "filters": {"concern": ["security"]}, "description": "Security vulnerabilities and attack vectors", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 65.2}, {"id": "security_critical", "label": "Security Critical", "filters": {"concern": ["security"], "risk": ["high", "critical"]}, "description": "Security vulnerabilities and attack vectors Significant impact, potential data loss.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 65.2}, {"id": "language_ruby", "label": "Best for Ruby", "filters": {"language": ["Ruby"]}, "description": "Ruby codebases with Rails patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 63.8}, {"id": "complexity_moderate", "label": "Best for Moderate Code", "filters": {"complexity": ["moderate"]}, "description": "Some abstraction.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 63.4}, {"id": "change_type_feature", "label": "Best for Features", "filters": {"change_type": ["feature"]}, "description": "New feature implementation", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 62.9}, {"id": "risk_medium", "label": "Best for Medium Risk", "filters": {"risk": ["medium"]}, "description": "Moderate user impact.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 59.5}];
+        const allModelsData = {"anthropic_claude-opus-4-5-20251101": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 4, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 3, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "sourcery": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 3}, "core": {"tp": 1, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 2}, "core": {"tp": 2, "fp": 6, "fn": 2}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "bugbot": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 7, "fn": 0}, "core": {"tp": 2, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 5}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 5}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 4, "fp": 13, "fn": 2}, "core": {"tp": 4, "fp": 13, "fn": 2}, "all": {"tp": 4, "fp": 13, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 3}, "core": {"tp": 3, "fp": 5, "fn": 3}, "all": {"tp": 3, "fp": 5, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 6}, "core": {"tp": 0, "fp": 2, "fn": 6}, "all": {"tp": 0, "fp": 2, "fn": 6}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 0}, "core": {"tp": 4, "fp": 1, "fn": 0}, "all": {"tp": 4, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 8, "fn": 1}, "core": {"tp": 3, "fp": 8, "fn": 1}, "all": {"tp": 3, "fp": 8, "fn": 1}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 3, "fp": 7, "fn": 1}, "core": {"tp": 3, "fp": 7, "fn": 1}, "all": {"tp": 3, "fp": 7, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 2}, "all": {"tp": 1, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 13, "fn": 1}, "core": {"tp": 2, "fp": 13, "fn": 1}, "all": {"tp": 2, "fp": 13, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 1}}, "claude": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 10, "fn": 2}, "core": {"tp": 0, "fp": 10, "fn": 2}, "all": {"tp": 0, "fp": 10, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "coderabbit": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 4}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 5}, "core": {"tp": 2, "fp": 0, "fn": 5}, "all": {"tp": 2, "fp": 0, "fn": 5}}, "codeant-v2": {"strict": {"tp": 4, "fp": 5, "fn": 3}, "core": {"tp": 4, "fp": 5, "fn": 3}, "all": {"tp": 4, "fp": 5, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 8, "fn": 4}, "core": {"tp": 3, "fp": 8, "fn": 4}, "all": {"tp": 3, "fp": 8, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 5}, "core": {"tp": 2, "fp": 2, "fn": 5}, "all": {"tp": 2, "fp": 2, "fn": 5}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 3}, "core": {"tp": 4, "fp": 2, "fn": 3}, "all": {"tp": 4, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "copilot-v2": {"strict": {"tp": 6, "fp": 5, "fn": 1}, "core": {"tp": 6, "fp": 5, "fn": 1}, "all": {"tp": 6, "fp": 5, "fn": 1}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "bugbot": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 6}, "core": {"tp": 1, "fp": 2, "fn": 6}, "all": {"tp": 1, "fp": 2, "fn": 6}}, "claude-code": {"strict": {"tp": 2, "fp": 4, "fn": 5}, "core": {"tp": 2, "fp": 4, "fn": 5}, "all": {"tp": 2, "fp": 4, "fn": 5}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 6}, "core": {"tp": 1, "fp": 1, "fn": 6}, "all": {"tp": 1, "fp": 1, "fn": 6}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 7}, "core": {"tp": 0, "fp": 0, "fn": 7}, "all": {"tp": 0, "fp": 0, "fn": 7}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 6}, "core": {"tp": 1, "fp": 1, "fn": 6}, "all": {"tp": 1, "fp": 1, "fn": 6}}, "kodus-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "macroscope": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 4, "fn": 4}, "core": {"tp": 3, "fp": 4, "fn": 4}, "all": {"tp": 3, "fp": 4, "fn": 4}}, "sourcery": {"strict": {"tp": 5, "fp": 7, "fn": 2}, "core": {"tp": 5, "fp": 7, "fn": 2}, "all": {"tp": 5, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 8, "fn": 3}, "core": {"tp": 0, "fp": 8, "fn": 3}, "all": {"tp": 1, "fp": 8, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 5, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 4, "fp": 6, "fn": 2}, "core": {"tp": 4, "fp": 6, "fn": 2}, "all": {"tp": 4, "fp": 6, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 9, "fn": 3}, "core": {"tp": 3, "fp": 9, "fn": 3}, "all": {"tp": 3, "fp": 9, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 1, "fp": 6, "fn": 5}, "core": {"tp": 1, "fp": 6, "fn": 5}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "sourcery": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 6}, "core": {"tp": 2, "fp": 2, "fn": 6}, "all": {"tp": 2, "fp": 2, "fn": 6}}, "codeant-v2": {"strict": {"tp": 3, "fp": 16, "fn": 5}, "core": {"tp": 3, "fp": 16, "fn": 5}, "all": {"tp": 3, "fp": 16, "fn": 5}}, "coderabbit": {"strict": {"tp": 6, "fp": 19, "fn": 2}, "core": {"tp": 6, "fp": 19, "fn": 2}, "all": {"tp": 6, "fp": 19, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 5, "fp": 4, "fn": 3}, "core": {"tp": 5, "fp": 4, "fn": 3}, "all": {"tp": 5, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 5, "fp": 0, "fn": 3}, "core": {"tp": 5, "fp": 0, "fn": 3}, "all": {"tp": 5, "fp": 0, "fn": 3}}, "copilot-v2": {"strict": {"tp": 6, "fp": 2, "fn": 2}, "core": {"tp": 6, "fp": 2, "fn": 2}, "all": {"tp": 6, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 5, "fp": 0, "fn": 3}, "core": {"tp": 5, "fp": 0, "fn": 3}, "all": {"tp": 5, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 4}, "core": {"tp": 4, "fp": 3, "fn": 4}, "all": {"tp": 4, "fp": 3, "fn": 4}}, "claude-code": {"strict": {"tp": 5, "fp": 0, "fn": 3}, "core": {"tp": 5, "fp": 0, "fn": 3}, "all": {"tp": 5, "fp": 0, "fn": 3}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kodus-v2": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 6}, "core": {"tp": 2, "fp": 4, "fn": 6}, "all": {"tp": 2, "fp": 4, "fn": 6}}, "qodo-v2": {"strict": {"tp": 6, "fp": 5, "fn": 2}, "core": {"tp": 6, "fp": 5, "fn": 2}, "all": {"tp": 6, "fp": 5, "fn": 2}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "codeant-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 9, "fn": 2}, "core": {"tp": 2, "fp": 9, "fn": 2}, "all": {"tp": 2, "fp": 9, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "copilot-v2": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 6, "fp": 0, "fn": 3}, "core": {"tp": 6, "fp": 0, "fn": 3}, "all": {"tp": 6, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 7, "fp": 4, "fn": 2}, "core": {"tp": 7, "fp": 4, "fn": 2}, "all": {"tp": 7, "fp": 4, "fn": 2}}, "coderabbit": {"strict": {"tp": 9, "fp": 10, "fn": 0}, "core": {"tp": 9, "fp": 10, "fn": 0}, "all": {"tp": 9, "fp": 10, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 7, "fp": 2, "fn": 2}, "core": {"tp": 7, "fp": 2, "fn": 2}, "all": {"tp": 7, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 2, "fn": 4}, "core": {"tp": 5, "fp": 2, "fn": 4}, "all": {"tp": 5, "fp": 2, "fn": 4}}, "gemini-v2": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 6, "fp": 2, "fn": 3}, "core": {"tp": 6, "fp": 2, "fn": 3}, "all": {"tp": 6, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 6, "fp": 2, "fn": 3}, "core": {"tp": 6, "fp": 2, "fn": 3}, "all": {"tp": 6, "fp": 2, "fn": 3}}, "augment": {"strict": {"tp": 9, "fp": 1, "fn": 0}, "core": {"tp": 9, "fp": 1, "fn": 0}, "all": {"tp": 9, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 6, "fp": 4, "fn": 3}, "core": {"tp": 6, "fp": 4, "fn": 3}, "all": {"tp": 6, "fp": 4, "fn": 3}}, "claude": {"strict": {"tp": 6, "fp": 0, "fn": 3}, "core": {"tp": 6, "fp": 0, "fn": 3}, "all": {"tp": 6, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 7, "fp": 0, "fn": 2}, "core": {"tp": 7, "fp": 0, "fn": 2}, "all": {"tp": 7, "fp": 0, "fn": 2}}, "cubic-v2": {"strict": {"tp": 4, "fp": 0, "fn": 5}, "core": {"tp": 4, "fp": 0, "fn": 5}, "all": {"tp": 4, "fp": 0, "fn": 5}}, "devin": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 8}, "core": {"tp": 1, "fp": 0, "fn": 8}, "all": {"tp": 1, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 9}, "core": {"tp": 0, "fp": 0, "fn": 9}, "all": {"tp": 0, "fp": 0, "fn": 9}}, "kodus-v2": {"strict": {"tp": 4, "fp": 3, "fn": 5}, "core": {"tp": 4, "fp": 3, "fn": 5}, "all": {"tp": 4, "fp": 3, "fn": 5}}, "macroscope": {"strict": {"tp": 7, "fp": 6, "fn": 2}, "core": {"tp": 7, "fp": 6, "fn": 2}, "all": {"tp": 7, "fp": 6, "fn": 2}}, "qodo-v2": {"strict": {"tp": 8, "fp": 0, "fn": 1}, "core": {"tp": 8, "fp": 0, "fn": 1}, "all": {"tp": 8, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 9, "fn": 1}, "core": {"tp": 2, "fp": 9, "fn": 1}, "all": {"tp": 2, "fp": 9, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "coderabbit": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "copilot-v2": {"strict": {"tp": 5, "fp": 3, "fn": 1}, "core": {"tp": 5, "fp": 3, "fn": 1}, "all": {"tp": 5, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 4, "fp": 2, "fn": 2}, "core": {"tp": 4, "fp": 2, "fn": 2}, "all": {"tp": 4, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "qodo-v2": {"strict": {"tp": 5, "fp": 2, "fn": 1}, "core": {"tp": 5, "fp": 2, "fn": 1}, "all": {"tp": 5, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 4, "fp": 4, "fn": 0}, "all": {"tp": 5, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 3}, "core": {"tp": 2, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 5, "fp": 7, "fn": 0}, "core": {"tp": 5, "fp": 7, "fn": 0}, "all": {"tp": 5, "fp": 7, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 4, "fn": 3}, "core": {"tp": 2, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 4}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 5, "fp": 6, "fn": 0}, "core": {"tp": 5, "fp": 6, "fn": 0}, "all": {"tp": 5, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 0, "fn": 1}, "core": {"tp": 4, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 5, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "devin": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 2, "fp": 6, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 10, "fn": 0}, "core": {"tp": 1, "fp": 10, "fn": 0}, "all": {"tp": 2, "fp": 10, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}], "tools": ["augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"greptile-v4-1": {"strict": {"precision": 49.0, "recall": 54.0, "f1": 51.4, "f2": 52.9, "tp": 75, "fp": 78, "fn": 64, "num_prs": 50}, "core": {"precision": 50.9, "recall": 51.3, "f1": 51.1, "f2": 51.2, "tp": 81, "fp": 78, "fn": 77, "num_prs": 50}, "all": {"precision": 53.3, "recall": 51.4, "f1": 52.4, "f2": 51.8, "tp": 89, "fp": 78, "fn": 84, "num_prs": 50}}, "codeant-v2": {"strict": {"precision": 39.0, "recall": 44.6, "f1": 41.6, "f2": 43.4, "tp": 62, "fp": 97, "fn": 77, "num_prs": 50}, "core": {"precision": 40.1, "recall": 41.1, "f1": 40.6, "f2": 40.9, "tp": 65, "fp": 97, "fn": 93, "num_prs": 50}, "all": {"precision": 41.2, "recall": 39.3, "f1": 40.2, "f2": 39.7, "tp": 68, "fp": 97, "fn": 105, "num_prs": 50}}, "baz": {"strict": {"precision": 60.6, "recall": 41.0, "f1": 48.9, "f2": 43.8, "tp": 57, "fp": 37, "fn": 82, "num_prs": 50}, "core": {"precision": 61.5, "recall": 37.3, "f1": 46.5, "f2": 40.5, "tp": 59, "fp": 37, "fn": 99, "num_prs": 50}, "all": {"precision": 62.6, "recall": 35.8, "f1": 45.6, "f2": 39.2, "tp": 62, "fp": 37, "fn": 111, "num_prs": 50}}, "claude-code": {"strict": {"precision": 44.3, "recall": 50.4, "f1": 47.1, "f2": 49.0, "tp": 70, "fp": 88, "fn": 69, "num_prs": 50}, "core": {"precision": 46.3, "recall": 48.1, "f1": 47.2, "f2": 47.7, "tp": 76, "fp": 88, "fn": 82, "num_prs": 50}, "all": {"precision": 48.8, "recall": 48.6, "f1": 48.7, "f2": 48.6, "tp": 84, "fp": 88, "fn": 89, "num_prs": 50}}, "kg": {"strict": {"precision": 53.3, "recall": 17.3, "f1": 26.1, "f2": 20.0, "tp": 24, "fp": 21, "fn": 115, "num_prs": 50}, "core": {"precision": 54.3, "recall": 15.8, "f1": 24.5, "f2": 18.4, "tp": 25, "fp": 21, "fn": 133, "num_prs": 50}, "all": {"precision": 55.3, "recall": 15.0, "f1": 23.6, "f2": 17.6, "tp": 26, "fp": 21, "fn": 147, "num_prs": 50}}, "graphite": {"strict": {"precision": 100.0, "recall": 7.9, "f1": 14.7, "f2": 9.7, "tp": 11, "fp": 0, "fn": 128, "num_prs": 50}, "core": {"precision": 100.0, "recall": 7.6, "f1": 14.1, "f2": 9.3, "tp": 12, "fp": 0, "fn": 146, "num_prs": 50}, "all": {"precision": 100.0, "recall": 6.9, "f1": 13.0, "f2": 8.5, "tp": 12, "fp": 0, "fn": 161, "num_prs": 50}}, "bugbot": {"strict": {"precision": 54.8, "recall": 48.9, "f1": 51.7, "f2": 50.0, "tp": 68, "fp": 56, "fn": 71, "num_prs": 50}, "core": {"precision": 56.9, "recall": 46.8, "f1": 51.4, "f2": 48.6, "tp": 74, "fp": 56, "fn": 84, "num_prs": 50}, "all": {"precision": 58.2, "recall": 45.1, "f1": 50.8, "f2": 47.2, "tp": 78, "fp": 56, "fn": 95, "num_prs": 50}}, "devin": {"strict": {"precision": 67.4, "recall": 43.2, "f1": 52.6, "f2": 46.5, "tp": 60, "fp": 29, "fn": 79, "num_prs": 50}, "core": {"precision": 69.1, "recall": 41.1, "f1": 51.6, "f2": 44.8, "tp": 65, "fp": 29, "fn": 93, "num_prs": 50}, "all": {"precision": 70.1, "recall": 39.3, "f1": 50.4, "f2": 43.1, "tp": 68, "fp": 29, "fn": 105, "num_prs": 50}}, "gitlab": {"strict": {"precision": 51.0, "recall": 54.0, "f1": 52.4, "f2": 53.3, "tp": 75, "fp": 72, "fn": 64, "num_prs": 50}, "core": {"precision": 53.5, "recall": 52.5, "f1": 53.0, "f2": 52.7, "tp": 83, "fp": 72, "fn": 75, "num_prs": 50}, "all": {"precision": 55.8, "recall": 52.6, "f1": 54.2, "f2": 53.2, "tp": 91, "fp": 72, "fn": 82, "num_prs": 50}}, "claude": {"strict": {"precision": 41.0, "recall": 39.6, "f1": 40.3, "f2": 39.9, "tp": 55, "fp": 79, "fn": 84, "num_prs": 50}, "core": {"precision": 43.2, "recall": 38.0, "f1": 40.4, "f2": 38.9, "tp": 60, "fp": 79, "fn": 98, "num_prs": 50}, "all": {"precision": 45.5, "recall": 38.2, "f1": 41.5, "f2": 39.4, "tp": 66, "fp": 79, "fn": 107, "num_prs": 50}}, "copilot-v2": {"strict": {"precision": 35.6, "recall": 66.9, "f1": 46.5, "f2": 56.9, "tp": 93, "fp": 168, "fn": 46, "num_prs": 50}, "core": {"precision": 38.5, "recall": 66.5, "f1": 48.7, "f2": 58.0, "tp": 105, "fp": 168, "fn": 53, "num_prs": 50}, "all": {"precision": 40.6, "recall": 66.5, "f1": 50.4, "f2": 59.0, "tp": 115, "fp": 168, "fn": 58, "num_prs": 50}}, "qodo-v2": {"strict": {"precision": 53.0, "recall": 64.0, "f1": 58.0, "f2": 61.5, "tp": 89, "fp": 79, "fn": 50, "num_prs": 50}, "core": {"precision": 55.4, "recall": 62.0, "f1": 58.5, "f2": 60.6, "tp": 98, "fp": 79, "fn": 60, "num_prs": 50}, "all": {"precision": 56.4, "recall": 59.0, "f1": 57.6, "f2": 58.4, "tp": 102, "fp": 79, "fn": 71, "num_prs": 50}}, "macroscope": {"strict": {"precision": 60.2, "recall": 53.2, "f1": 56.5, "f2": 54.5, "tp": 74, "fp": 49, "fn": 65, "num_prs": 50}, "core": {"precision": 61.1, "recall": 48.7, "f1": 54.2, "f2": 50.8, "tp": 77, "fp": 49, "fn": 81, "num_prs": 50}, "all": {"precision": 62.3, "recall": 46.8, "f1": 53.5, "f2": 49.3, "tp": 81, "fp": 49, "fn": 92, "num_prs": 50}}, "qodo-extended-v2": {"strict": {"precision": 64.8, "recall": 66.2, "f1": 65.5, "f2": 65.9, "tp": 92, "fp": 50, "fn": 47, "num_prs": 50}, "core": {"precision": 67.1, "recall": 64.6, "f1": 65.8, "f2": 65.1, "tp": 102, "fp": 50, "fn": 56, "num_prs": 50}, "all": {"precision": 67.9, "recall": 61.3, "f1": 64.4, "f2": 62.5, "tp": 106, "fp": 50, "fn": 67, "num_prs": 50}}, "cubic-v2": {"strict": {"precision": 58.3, "recall": 65.5, "f1": 61.7, "f2": 63.9, "tp": 91, "fp": 65, "fn": 48, "num_prs": 50}, "core": {"precision": 61.5, "recall": 65.8, "f1": 63.6, "f2": 64.9, "tp": 104, "fp": 65, "fn": 54, "num_prs": 50}, "all": {"precision": 62.4, "recall": 62.4, "f1": 62.4, "f2": 62.4, "tp": 108, "fp": 65, "fn": 65, "num_prs": 50}}, "kodus-v2": {"strict": {"precision": 57.1, "recall": 40.3, "f1": 47.3, "f2": 42.8, "tp": 56, "fp": 42, "fn": 83, "num_prs": 50}, "core": {"precision": 59.2, "recall": 38.6, "f1": 46.7, "f2": 41.5, "tp": 61, "fp": 42, "fn": 97, "num_prs": 50}, "all": {"precision": 59.6, "recall": 35.8, "f1": 44.8, "f2": 38.9, "tp": 62, "fp": 42, "fn": 111, "num_prs": 50}}, "gemini-v2": {"strict": {"precision": 43.2, "recall": 57.6, "f1": 49.4, "f2": 54.0, "tp": 80, "fp": 105, "fn": 59, "num_prs": 50}, "core": {"precision": 45.3, "recall": 55.1, "f1": 49.7, "f2": 52.8, "tp": 87, "fp": 105, "fn": 71, "num_prs": 50}, "all": {"precision": 47.5, "recall": 54.9, "f1": 50.9, "f2": 53.3, "tp": 95, "fp": 105, "fn": 78, "num_prs": 50}}, "sourcery": {"strict": {"precision": 38.4, "recall": 56.1, "f1": 45.6, "f2": 51.4, "tp": 78, "fp": 125, "fn": 61, "num_prs": 50}, "core": {"precision": 41.0, "recall": 55.1, "f1": 47.0, "f2": 51.5, "tp": 87, "fp": 125, "fn": 71, "num_prs": 50}, "all": {"precision": 43.2, "recall": 54.9, "f1": 48.3, "f2": 52.1, "tp": 95, "fp": 125, "fn": 78, "num_prs": 50}}, "augment": {"strict": {"precision": 56.5, "recall": 65.5, "f1": 60.7, "f2": 63.5, "tp": 91, "fp": 70, "fn": 48, "num_prs": 50}, "core": {"precision": 59.5, "recall": 65.2, "f1": 62.2, "f2": 64.0, "tp": 103, "fp": 70, "fn": 55, "num_prs": 50}, "all": {"precision": 60.7, "recall": 62.4, "f1": 61.5, "f2": 62.1, "tp": 108, "fp": 70, "fn": 65, "num_prs": 50}}, "coderabbit": {"strict": {"precision": 31.0, "recall": 62.6, "f1": 41.4, "f2": 52.0, "tp": 87, "fp": 194, "fn": 52, "num_prs": 50}, "core": {"precision": 32.6, "recall": 59.5, "f1": 42.2, "f2": 51.1, "tp": 94, "fp": 194, "fn": 64, "num_prs": 50}, "all": {"precision": 34.7, "recall": 59.5, "f1": 43.8, "f2": 52.1, "tp": 103, "fp": 194, "fn": 70, "num_prs": 50}}}}, "anthropic_claude-sonnet-4-5-20250929": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 4, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "sourcery": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 6, "fn": 0}, "core": {"tp": 4, "fp": 6, "fn": 0}, "all": {"tp": 4, "fp": 6, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 1, "fp": 6, "fn": 4}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 8, "fn": 0}, "core": {"tp": 2, "fp": 8, "fn": 0}, "all": {"tp": 2, "fp": 8, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 5}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 4}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 1, "fp": 5, "fn": 2}, "core": {"tp": 3, "fp": 5, "fn": 2}, "all": {"tp": 3, "fp": 5, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 6}, "core": {"tp": 0, "fp": 2, "fn": 6}, "all": {"tp": 0, "fp": 2, "fn": 6}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 4, "fp": 13, "fn": 2}, "core": {"tp": 4, "fp": 13, "fn": 2}, "all": {"tp": 4, "fp": 13, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 5}, "core": {"tp": 1, "fp": 3, "fn": 5}, "all": {"tp": 1, "fp": 3, "fn": 5}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 3}, "core": {"tp": 3, "fp": 5, "fn": 3}, "all": {"tp": 3, "fp": 5, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 7, "fn": 2}, "core": {"tp": 2, "fp": 7, "fn": 2}, "all": {"tp": 2, "fp": 7, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 2}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 1, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 0}, "core": {"tp": 4, "fp": 1, "fn": 0}, "all": {"tp": 4, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 10, "fn": 1}, "core": {"tp": 3, "fp": 10, "fn": 1}, "all": {"tp": 3, "fp": 10, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 7, "fn": 2}, "core": {"tp": 1, "fp": 7, "fn": 2}, "all": {"tp": 1, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 0, "fp": 5, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 8, "fn": 2}, "core": {"tp": 1, "fp": 8, "fn": 2}, "all": {"tp": 1, "fp": 8, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 13, "fn": 1}, "core": {"tp": 2, "fp": 13, "fn": 1}, "all": {"tp": 2, "fp": 13, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 2, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "claude-code": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 3, "fp": 4, "fn": 0}}, "codeant-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 6, "fn": 0}, "core": {"tp": 3, "fp": 6, "fn": 0}, "all": {"tp": 3, "fp": 6, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 7, "fn": 2}, "core": {"tp": 1, "fp": 7, "fn": 2}, "all": {"tp": 1, "fp": 7, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 6, "fn": 0}, "core": {"tp": 3, "fp": 6, "fn": 0}, "all": {"tp": 3, "fp": 6, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 12, "fn": 2}, "core": {"tp": 0, "fp": 12, "fn": 2}, "all": {"tp": 0, "fp": 12, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "coderabbit": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 2, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 4, "fn": 4}, "core": {"tp": 0, "fp": 4, "fn": 5}, "all": {"tp": 0, "fp": 4, "fn": 5}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 4}, "core": {"tp": 3, "fp": 2, "fn": 4}, "all": {"tp": 3, "fp": 2, "fn": 4}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 5}, "core": {"tp": 2, "fp": 0, "fn": 5}, "all": {"tp": 2, "fp": 0, "fn": 5}}, "bugbot": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 6}, "core": {"tp": 1, "fp": 2, "fn": 6}, "all": {"tp": 1, "fp": 2, "fn": 6}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 6}, "core": {"tp": 1, "fp": 5, "fn": 6}, "all": {"tp": 1, "fp": 5, "fn": 6}}, "codeant-v2": {"strict": {"tp": 2, "fp": 7, "fn": 5}, "core": {"tp": 2, "fp": 7, "fn": 5}, "all": {"tp": 2, "fp": 7, "fn": 5}}, "coderabbit": {"strict": {"tp": 4, "fp": 10, "fn": 3}, "core": {"tp": 4, "fp": 10, "fn": 3}, "all": {"tp": 4, "fp": 10, "fn": 3}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 6}, "core": {"tp": 1, "fp": 1, "fn": 6}, "all": {"tp": 1, "fp": 1, "fn": 6}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 7}, "core": {"tp": 0, "fp": 0, "fn": 7}, "all": {"tp": 0, "fp": 0, "fn": 7}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 6}, "core": {"tp": 1, "fp": 0, "fn": 6}, "all": {"tp": 1, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 5, "fp": 3, "fn": 2}, "core": {"tp": 5, "fp": 3, "fn": 2}, "all": {"tp": 5, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 4, "fn": 4}, "core": {"tp": 3, "fp": 4, "fn": 4}, "all": {"tp": 3, "fp": 4, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 5, "fn": 5}, "core": {"tp": 2, "fp": 5, "fn": 5}, "all": {"tp": 2, "fp": 5, "fn": 5}}, "sourcery": {"strict": {"tp": 4, "fp": 7, "fn": 3}, "core": {"tp": 4, "fp": 7, "fn": 3}, "all": {"tp": 4, "fp": 7, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 5}, "core": {"tp": 2, "fp": 2, "fn": 5}, "all": {"tp": 2, "fp": 2, "fn": 5}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 3}, "core": {"tp": 4, "fp": 2, "fn": 3}, "all": {"tp": 4, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "copilot-v2": {"strict": {"tp": 6, "fp": 5, "fn": 1}, "core": {"tp": 6, "fp": 5, "fn": 1}, "all": {"tp": 6, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 9, "fn": 3}, "core": {"tp": 0, "fp": 9, "fn": 3}, "all": {"tp": 0, "fp": 9, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "codeant-v2": {"strict": {"tp": 4, "fp": 9, "fn": 2}, "core": {"tp": 4, "fp": 9, "fn": 2}, "all": {"tp": 4, "fp": 9, "fn": 2}}, "coderabbit": {"strict": {"tp": 4, "fp": 6, "fn": 2}, "core": {"tp": 4, "fp": 6, "fn": 2}, "all": {"tp": 4, "fp": 6, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 1, "fp": 6, "fn": 5}, "core": {"tp": 1, "fp": 6, "fn": 5}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 3, "fn": 5}, "core": {"tp": 1, "fp": 3, "fn": 5}, "all": {"tp": 1, "fp": 3, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 3, "fp": 14, "fn": 3}, "core": {"tp": 3, "fp": 14, "fn": 3}, "all": {"tp": 3, "fp": 14, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 2}, "all": {"tp": 1, "fp": 6, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "baz": {"strict": {"tp": 2, "fp": 2, "fn": 6}, "core": {"tp": 2, "fp": 2, "fn": 6}, "all": {"tp": 2, "fp": 2, "fn": 6}}, "bugbot": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}, "claude": {"strict": {"tp": 6, "fp": 2, "fn": 2}, "core": {"tp": 6, "fp": 2, "fn": 2}, "all": {"tp": 6, "fp": 2, "fn": 2}}, "claude-code": {"strict": {"tp": 4, "fp": 0, "fn": 4}, "core": {"tp": 4, "fp": 0, "fn": 4}, "all": {"tp": 4, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 3, "fp": 16, "fn": 5}, "core": {"tp": 3, "fp": 16, "fn": 5}, "all": {"tp": 3, "fp": 16, "fn": 5}}, "coderabbit": {"strict": {"tp": 5, "fp": 11, "fn": 3}, "core": {"tp": 5, "fp": 11, "fn": 3}, "all": {"tp": 5, "fp": 11, "fn": 3}}, "cubic-v2": {"strict": {"tp": 1, "fp": 4, "fn": 7}, "core": {"tp": 1, "fp": 4, "fn": 7}, "all": {"tp": 1, "fp": 4, "fn": 7}}, "devin": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kodus-v2": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "macroscope": {"strict": {"tp": 2, "fp": 5, "fn": 6}, "core": {"tp": 2, "fp": 5, "fn": 6}, "all": {"tp": 2, "fp": 5, "fn": 6}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "qodo-v2": {"strict": {"tp": 6, "fp": 3, "fn": 2}, "core": {"tp": 6, "fp": 3, "fn": 2}, "all": {"tp": 6, "fp": 3, "fn": 2}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 1, "fn": 3}, "core": {"tp": 5, "fp": 1, "fn": 3}, "all": {"tp": 5, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 4, "fp": 6, "fn": 4}, "core": {"tp": 4, "fp": 6, "fn": 4}, "all": {"tp": 4, "fp": 6, "fn": 4}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 5, "fp": 5, "fn": 3}, "core": {"tp": 5, "fp": 5, "fn": 3}, "all": {"tp": 5, "fp": 5, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 5, "fn": 4}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 0, "fp": 5, "fn": 4}}, "codeant-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kg": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 8, "fp": 2, "fn": 1}, "core": {"tp": 8, "fp": 2, "fn": 1}, "all": {"tp": 8, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "bugbot": {"strict": {"tp": 6, "fp": 1, "fn": 3}, "core": {"tp": 6, "fp": 1, "fn": 3}, "all": {"tp": 6, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 4, "fp": 0, "fn": 5}, "core": {"tp": 4, "fp": 0, "fn": 5}, "all": {"tp": 4, "fp": 0, "fn": 5}}, "codeant-v2": {"strict": {"tp": 4, "fp": 4, "fn": 5}, "core": {"tp": 4, "fp": 4, "fn": 5}, "all": {"tp": 4, "fp": 4, "fn": 5}}, "coderabbit": {"strict": {"tp": 9, "fp": 9, "fn": 0}, "core": {"tp": 9, "fp": 9, "fn": 0}, "all": {"tp": 9, "fp": 9, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 6}, "core": {"tp": 3, "fp": 0, "fn": 6}, "all": {"tp": 3, "fp": 0, "fn": 6}}, "devin": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 8}, "core": {"tp": 1, "fp": 0, "fn": 8}, "all": {"tp": 1, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 9}, "core": {"tp": 0, "fp": 0, "fn": 9}, "all": {"tp": 0, "fp": 0, "fn": 9}}, "kodus-v2": {"strict": {"tp": 4, "fp": 3, "fn": 5}, "core": {"tp": 4, "fp": 3, "fn": 5}, "all": {"tp": 4, "fp": 3, "fn": 5}}, "macroscope": {"strict": {"tp": 5, "fp": 5, "fn": 4}, "core": {"tp": 5, "fp": 5, "fn": 4}, "all": {"tp": 5, "fp": 5, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 7, "fp": 2, "fn": 2}, "core": {"tp": 7, "fp": 2, "fn": 2}, "all": {"tp": 7, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 8, "fp": 3, "fn": 1}, "core": {"tp": 8, "fp": 3, "fn": 1}, "all": {"tp": 8, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 3, "fn": 4}, "core": {"tp": 5, "fp": 3, "fn": 4}, "all": {"tp": 5, "fp": 3, "fn": 4}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 5}, "core": {"tp": 4, "fp": 2, "fn": 5}, "all": {"tp": 4, "fp": 2, "fn": 5}}, "gitlab": {"strict": {"tp": 5, "fp": 2, "fn": 4}, "core": {"tp": 5, "fp": 2, "fn": 4}, "all": {"tp": 5, "fp": 2, "fn": 4}}, "copilot-v2": {"strict": {"tp": 6, "fp": 1, "fn": 3}, "core": {"tp": 6, "fp": 1, "fn": 3}, "all": {"tp": 6, "fp": 1, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "codeant-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "coderabbit": {"strict": {"tp": 4, "fp": 4, "fn": 2}, "core": {"tp": 4, "fp": 4, "fn": 2}, "all": {"tp": 4, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 4, "fp": 2, "fn": 2}, "core": {"tp": 4, "fp": 2, "fn": 2}, "all": {"tp": 4, "fp": 2, "fn": 2}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "copilot-v2": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 4, "fp": 4, "fn": 0}, "all": {"tp": 5, "fp": 4, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 2, "fp": 4, "fn": 3}, "core": {"tp": 2, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 4}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 5, "fn": 2}, "core": {"tp": 3, "fp": 5, "fn": 2}, "all": {"tp": 3, "fp": 5, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 7, "fn": 4}, "core": {"tp": 1, "fp": 7, "fn": 4}, "all": {"tp": 1, "fp": 7, "fn": 5}}, "coderabbit": {"strict": {"tp": 5, "fp": 5, "fn": 0}, "core": {"tp": 5, "fp": 5, "fn": 0}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 3, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 4, "fn": 4}, "core": {"tp": 1, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 5}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 5, "fn": 3}, "core": {"tp": 2, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "gemini-v2": {"strict": {"tp": 1, "fp": 6, "fn": 4}, "core": {"tp": 1, "fp": 6, "fn": 4}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "gitlab": {"strict": {"tp": 3, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "copilot-v2": {"strict": {"tp": 4, "fp": 7, "fn": 1}, "core": {"tp": 4, "fp": 7, "fn": 1}, "all": {"tp": 4, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 9, "fn": 0}, "core": {"tp": 1, "fp": 9, "fn": 0}, "all": {"tp": 2, "fp": 9, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}], "tools": ["augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"greptile-v4-1": {"strict": {"precision": 44.9, "recall": 50.4, "f1": 47.5, "f2": 49.2, "tp": 70, "fp": 86, "fn": 69, "num_prs": 50}, "core": {"precision": 46.9, "recall": 48.1, "f1": 47.5, "f2": 47.9, "tp": 76, "fp": 86, "fn": 82, "num_prs": 50}, "all": {"precision": 49.1, "recall": 48.0, "f1": 48.5, "f2": 48.2, "tp": 83, "fp": 86, "fn": 90, "num_prs": 50}}, "codeant-v2": {"strict": {"precision": 31.1, "recall": 37.4, "f1": 34.0, "f2": 36.0, "tp": 52, "fp": 115, "fn": 87, "num_prs": 50}, "core": {"precision": 32.4, "recall": 34.8, "f1": 33.5, "f2": 34.3, "tp": 55, "fp": 115, "fn": 103, "num_prs": 50}, "all": {"precision": 33.1, "recall": 32.9, "f1": 33.0, "f2": 33.0, "tp": 57, "fp": 115, "fn": 116, "num_prs": 50}}, "baz": {"strict": {"precision": 54.6, "recall": 38.1, "f1": 44.9, "f2": 40.6, "tp": 53, "fp": 44, "fn": 86, "num_prs": 50}, "core": {"precision": 55.6, "recall": 34.8, "f1": 42.8, "f2": 37.6, "tp": 55, "fp": 44, "fn": 103, "num_prs": 50}, "all": {"precision": 56.9, "recall": 33.5, "f1": 42.2, "f2": 36.5, "tp": 58, "fp": 44, "fn": 115, "num_prs": 50}}, "claude-code": {"strict": {"precision": 34.3, "recall": 42.4, "f1": 37.9, "f2": 40.5, "tp": 59, "fp": 113, "fn": 80, "num_prs": 50}, "core": {"precision": 36.5, "recall": 41.1, "f1": 38.7, "f2": 40.1, "tp": 65, "fp": 113, "fn": 93, "num_prs": 50}, "all": {"precision": 38.3, "recall": 40.5, "f1": 39.3, "f2": 40.0, "tp": 70, "fp": 113, "fn": 103, "num_prs": 50}}, "kg": {"strict": {"precision": 48.8, "recall": 15.1, "f1": 23.1, "f2": 17.5, "tp": 21, "fp": 22, "fn": 118, "num_prs": 50}, "core": {"precision": 50.0, "recall": 13.9, "f1": 21.8, "f2": 16.3, "tp": 22, "fp": 22, "fn": 136, "num_prs": 50}, "all": {"precision": 51.1, "recall": 13.3, "f1": 21.1, "f2": 15.6, "tp": 23, "fp": 22, "fn": 150, "num_prs": 50}}, "graphite": {"strict": {"precision": 100.0, "recall": 7.9, "f1": 14.7, "f2": 9.7, "tp": 11, "fp": 0, "fn": 128, "num_prs": 50}, "core": {"precision": 100.0, "recall": 7.6, "f1": 14.1, "f2": 9.3, "tp": 12, "fp": 0, "fn": 146, "num_prs": 50}, "all": {"precision": 100.0, "recall": 6.9, "f1": 13.0, "f2": 8.5, "tp": 12, "fp": 0, "fn": 161, "num_prs": 50}}, "bugbot": {"strict": {"precision": 52.0, "recall": 46.0, "f1": 48.9, "f2": 47.1, "tp": 64, "fp": 59, "fn": 75, "num_prs": 50}, "core": {"precision": 54.3, "recall": 44.3, "f1": 48.8, "f2": 46.0, "tp": 70, "fp": 59, "fn": 88, "num_prs": 50}, "all": {"precision": 55.3, "recall": 42.2, "f1": 47.9, "f2": 44.3, "tp": 73, "fp": 59, "fn": 100, "num_prs": 50}}, "devin": {"strict": {"precision": 66.0, "recall": 44.6, "f1": 53.2, "f2": 47.7, "tp": 62, "fp": 32, "fn": 77, "num_prs": 50}, "core": {"precision": 67.3, "recall": 41.8, "f1": 51.6, "f2": 45.2, "tp": 66, "fp": 32, "fn": 92, "num_prs": 50}, "all": {"precision": 68.3, "recall": 39.9, "f1": 50.4, "f2": 43.5, "tp": 69, "fp": 32, "fn": 104, "num_prs": 50}}, "gitlab": {"strict": {"precision": 44.7, "recall": 48.9, "f1": 46.7, "f2": 48.0, "tp": 68, "fp": 84, "fn": 71, "num_prs": 50}, "core": {"precision": 47.5, "recall": 48.1, "f1": 47.8, "f2": 48.0, "tp": 76, "fp": 84, "fn": 82, "num_prs": 50}, "all": {"precision": 49.7, "recall": 48.0, "f1": 48.8, "f2": 48.3, "tp": 83, "fp": 84, "fn": 90, "num_prs": 50}}, "claude": {"strict": {"precision": 36.1, "recall": 37.4, "f1": 36.7, "f2": 37.1, "tp": 52, "fp": 92, "fn": 87, "num_prs": 50}, "core": {"precision": 38.3, "recall": 36.1, "f1": 37.1, "f2": 36.5, "tp": 57, "fp": 92, "fn": 101, "num_prs": 50}, "all": {"precision": 40.6, "recall": 36.4, "f1": 38.4, "f2": 37.2, "tp": 63, "fp": 92, "fn": 110, "num_prs": 50}}, "copilot-v2": {"strict": {"precision": 31.4, "recall": 61.9, "f1": 41.6, "f2": 51.8, "tp": 86, "fp": 188, "fn": 53, "num_prs": 50}, "core": {"precision": 34.3, "recall": 62.0, "f1": 44.1, "f2": 53.4, "tp": 98, "fp": 188, "fn": 60, "num_prs": 50}, "all": {"precision": 36.1, "recall": 61.3, "f1": 45.4, "f2": 53.8, "tp": 106, "fp": 188, "fn": 67, "num_prs": 50}}, "qodo-v2": {"strict": {"precision": 45.1, "recall": 59.0, "f1": 51.1, "f2": 55.6, "tp": 82, "fp": 100, "fn": 57, "num_prs": 50}, "core": {"precision": 47.1, "recall": 56.3, "f1": 51.3, "f2": 54.2, "tp": 89, "fp": 100, "fn": 69, "num_prs": 50}, "all": {"precision": 47.9, "recall": 53.2, "f1": 50.4, "f2": 52.0, "tp": 92, "fp": 100, "fn": 81, "num_prs": 50}}, "macroscope": {"strict": {"precision": 50.4, "recall": 45.3, "f1": 47.7, "f2": 46.3, "tp": 63, "fp": 62, "fn": 76, "num_prs": 50}, "core": {"precision": 51.6, "recall": 41.8, "f1": 46.2, "f2": 43.4, "tp": 66, "fp": 62, "fn": 92, "num_prs": 50}, "all": {"precision": 53.0, "recall": 40.5, "f1": 45.9, "f2": 42.5, "tp": 70, "fp": 62, "fn": 103, "num_prs": 50}}, "qodo-extended-v2": {"strict": {"precision": 53.6, "recall": 58.3, "f1": 55.9, "f2": 57.3, "tp": 81, "fp": 70, "fn": 58, "num_prs": 50}, "core": {"precision": 56.2, "recall": 57.0, "f1": 56.6, "f2": 56.8, "tp": 90, "fp": 70, "fn": 68, "num_prs": 50}, "all": {"precision": 56.8, "recall": 53.2, "f1": 54.9, "f2": 53.9, "tp": 92, "fp": 70, "fn": 81, "num_prs": 50}}, "cubic-v2": {"strict": {"precision": 52.9, "recall": 59.7, "f1": 56.1, "f2": 58.2, "tp": 83, "fp": 74, "fn": 56, "num_prs": 50}, "core": {"precision": 56.5, "recall": 60.8, "f1": 58.5, "f2": 59.9, "tp": 96, "fp": 74, "fn": 62, "num_prs": 50}, "all": {"precision": 57.2, "recall": 57.2, "f1": 57.2, "f2": 57.2, "tp": 99, "fp": 74, "fn": 74, "num_prs": 50}}, "kodus-v2": {"strict": {"precision": 55.2, "recall": 38.1, "f1": 45.1, "f2": 40.6, "tp": 53, "fp": 43, "fn": 86, "num_prs": 50}, "core": {"precision": 57.4, "recall": 36.7, "f1": 44.8, "f2": 39.6, "tp": 58, "fp": 43, "fn": 100, "num_prs": 50}, "all": {"precision": 57.8, "recall": 34.1, "f1": 42.9, "f2": 37.2, "tp": 59, "fp": 43, "fn": 114, "num_prs": 50}}, "sourcery": {"strict": {"precision": 34.2, "recall": 48.9, "f1": 40.2, "f2": 45.0, "tp": 68, "fp": 131, "fn": 71, "num_prs": 50}, "core": {"precision": 37.0, "recall": 48.7, "f1": 42.1, "f2": 45.8, "tp": 77, "fp": 131, "fn": 81, "num_prs": 50}, "all": {"precision": 38.8, "recall": 48.0, "f1": 42.9, "f2": 45.8, "tp": 83, "fp": 131, "fn": 90, "num_prs": 50}}, "gemini-v2": {"strict": {"precision": 36.7, "recall": 51.8, "f1": 43.0, "f2": 47.9, "tp": 72, "fp": 124, "fn": 67, "num_prs": 50}, "core": {"precision": 39.2, "recall": 50.6, "f1": 44.2, "f2": 47.8, "tp": 80, "fp": 124, "fn": 78, "num_prs": 50}, "all": {"precision": 41.2, "recall": 50.3, "f1": 45.3, "f2": 48.2, "tp": 87, "fp": 124, "fn": 86, "num_prs": 50}}, "augment": {"strict": {"precision": 49.4, "recall": 61.9, "f1": 55.0, "f2": 58.9, "tp": 86, "fp": 88, "fn": 53, "num_prs": 50}, "core": {"precision": 52.4, "recall": 61.4, "f1": 56.6, "f2": 59.4, "tp": 97, "fp": 88, "fn": 61, "num_prs": 50}, "all": {"precision": 53.7, "recall": 59.0, "f1": 56.2, "f2": 57.8, "tp": 102, "fp": 88, "fn": 71, "num_prs": 50}}, "coderabbit": {"strict": {"precision": 31.8, "recall": 60.4, "f1": 41.7, "f2": 51.2, "tp": 84, "fp": 180, "fn": 55, "num_prs": 50}, "core": {"precision": 33.8, "recall": 58.2, "f1": 42.8, "f2": 50.9, "tp": 92, "fp": 180, "fn": 66, "num_prs": 50}, "all": {"precision": 35.3, "recall": 56.6, "f1": 43.5, "f2": 50.5, "tp": 98, "fp": 180, "fn": 75, "num_prs": 50}}}}, "gpt-5.5": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 6, "fn": 3}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 0}, "all": {"tp": 0, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 0, "fp": 5, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 2}, "all": {"tp": 1, "fp": 6, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 2, "fp": 7, "fn": 0}, "core": {"tp": 2, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 3, "fp": 6, "fn": 2}, "core": {"tp": 3, "fp": 6, "fn": 2}, "all": {"tp": 3, "fp": 6, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 2, "fp": 6, "fn": 0}, "core": {"tp": 2, "fp": 6, "fn": 0}, "all": {"tp": 2, "fp": 6, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 4, "fp": 9, "fn": 0}, "core": {"tp": 4, "fp": 9, "fn": 0}, "all": {"tp": 4, "fp": 9, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 10, "fn": 2}, "core": {"tp": 1, "fp": 10, "fn": 2}, "all": {"tp": 2, "fp": 10, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 0, "fp": 6, "fn": 3}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"deputydev-v1": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}], "tools": ["deputydev-v1"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"deputydev-v1": {"strict": {"precision": 22.9, "recall": 50.6, "f1": 31.5, "f2": 40.7, "tp": 43, "fp": 145, "fn": 42, "num_prs": 50}, "core": {"precision": 25.3, "recall": 50.0, "f1": 33.6, "f2": 41.8, "tp": 49, "fp": 145, "fn": 49, "num_prs": 50}, "all": {"precision": 27.5, "recall": 50.0, "f1": 35.5, "f2": 43.0, "tp": 55, "fp": 145, "fn": 55, "num_prs": 50}}}}, "openai_gpt-5.2": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 4, "fp": 6, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 3, "fp": 5, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 3, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "sourcery": {"strict": {"tp": 0, "fp": 13, "fn": 1}, "core": {"tp": 0, "fp": 13, "fn": 3}, "all": {"tp": 2, "fp": 13, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 2}, "all": {"tp": 2, "fp": 5, "fn": 2}}, "coderabbit": {"strict": {"tp": 3, "fp": 7, "fn": 0}, "core": {"tp": 3, "fp": 7, "fn": 1}, "all": {"tp": 3, "fp": 7, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 3}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 8, "fn": 2}, "core": {"tp": 0, "fp": 8, "fn": 2}, "all": {"tp": 0, "fp": 8, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 8, "fn": 1}, "core": {"tp": 1, "fp": 8, "fn": 1}, "all": {"tp": 1, "fp": 8, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 7, "fn": 2}, "core": {"tp": 0, "fp": 7, "fn": 2}, "all": {"tp": 0, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 3}, "core": {"tp": 1, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 7, "fn": 3}, "core": {"tp": 1, "fp": 7, "fn": 3}, "all": {"tp": 1, "fp": 7, "fn": 4}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 8, "fn": 0}, "core": {"tp": 2, "fp": 8, "fn": 0}, "all": {"tp": 2, "fp": 8, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 4, "fp": 4, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 3, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 4}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 5}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 4, "fp": 13, "fn": 2}, "core": {"tp": 4, "fp": 13, "fn": 2}, "all": {"tp": 4, "fp": 13, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 3}, "core": {"tp": 3, "fp": 5, "fn": 3}, "all": {"tp": 3, "fp": 5, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 6}, "core": {"tp": 0, "fp": 2, "fn": 6}, "all": {"tp": 0, "fp": 2, "fn": 6}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 6, "fn": 2}, "core": {"tp": 2, "fp": 6, "fn": 2}, "all": {"tp": 2, "fp": 6, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 7, "fn": 1}, "core": {"tp": 1, "fp": 7, "fn": 1}, "all": {"tp": 1, "fp": 7, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 8, "fn": 0}, "core": {"tp": 0, "fp": 8, "fn": 1}, "all": {"tp": 0, "fp": 8, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 11, "fn": 0}, "core": {"tp": 0, "fp": 11, "fn": 1}, "all": {"tp": 0, "fp": 11, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 0}, "core": {"tp": 4, "fp": 1, "fn": 0}, "all": {"tp": 4, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 10, "fn": 1}, "core": {"tp": 3, "fp": 10, "fn": 1}, "all": {"tp": 3, "fp": 10, "fn": 1}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 3, "fp": 7, "fn": 1}, "core": {"tp": 3, "fp": 7, "fn": 1}, "all": {"tp": 3, "fp": 7, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 6, "fn": 3}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 9, "fn": 2}, "core": {"tp": 1, "fp": 9, "fn": 2}, "all": {"tp": 1, "fp": 9, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 13, "fn": 1}, "core": {"tp": 2, "fp": 13, "fn": 1}, "all": {"tp": 2, "fp": 13, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 8, "fn": 2}, "core": {"tp": 1, "fp": 8, "fn": 2}, "all": {"tp": 1, "fp": 8, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 2, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 3, "fp": 4, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 3, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 1}, "core": {"tp": 1, "fp": 7, "fn": 1}, "all": {"tp": 1, "fp": 7, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 9, "fn": 2}, "core": {"tp": 0, "fp": 9, "fn": 2}, "all": {"tp": 0, "fp": 9, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "coderabbit": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 4, "fp": 5, "fn": 1}, "all": {"tp": 4, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 4, "fn": 4}, "core": {"tp": 0, "fp": 4, "fn": 5}, "all": {"tp": 0, "fp": 4, "fn": 5}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 2}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 5}, "core": {"tp": 2, "fp": 1, "fn": 5}, "all": {"tp": 2, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 7, "fn": 5}, "core": {"tp": 2, "fp": 7, "fn": 5}, "all": {"tp": 2, "fp": 7, "fn": 5}}, "coderabbit": {"strict": {"tp": 5, "fp": 14, "fn": 2}, "core": {"tp": 5, "fp": 14, "fn": 2}, "all": {"tp": 5, "fp": 14, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 6, "fn": 3}, "core": {"tp": 4, "fp": 6, "fn": 3}, "all": {"tp": 4, "fp": 6, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 3}, "core": {"tp": 4, "fp": 2, "fn": 3}, "all": {"tp": 4, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "copilot-v2": {"strict": {"tp": 6, "fp": 5, "fn": 1}, "core": {"tp": 6, "fp": 5, "fn": 1}, "all": {"tp": 6, "fp": 5, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 1, "fn": 3}, "core": {"tp": 4, "fp": 1, "fn": 3}, "all": {"tp": 4, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 6}, "core": {"tp": 1, "fp": 2, "fn": 6}, "all": {"tp": 1, "fp": 2, "fn": 6}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 6}, "core": {"tp": 1, "fp": 5, "fn": 6}, "all": {"tp": 1, "fp": 5, "fn": 6}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 5}, "core": {"tp": 2, "fp": 0, "fn": 5}, "all": {"tp": 2, "fp": 0, "fn": 5}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 7}, "core": {"tp": 0, "fp": 0, "fn": 7}, "all": {"tp": 0, "fp": 0, "fn": 7}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 6}, "core": {"tp": 1, "fp": 0, "fn": 6}, "all": {"tp": 1, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "macroscope": {"strict": {"tp": 4, "fp": 3, "fn": 3}, "core": {"tp": 4, "fp": 3, "fn": 3}, "all": {"tp": 4, "fp": 3, "fn": 3}}, "qodo-v2": {"strict": {"tp": 4, "fp": 6, "fn": 3}, "core": {"tp": 4, "fp": 6, "fn": 3}, "all": {"tp": 4, "fp": 6, "fn": 3}}, "sourcery": {"strict": {"tp": 5, "fp": 7, "fn": 2}, "core": {"tp": 5, "fp": 7, "fn": 2}, "all": {"tp": 5, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 9, "fn": 3}, "core": {"tp": 0, "fp": 9, "fn": 3}, "all": {"tp": 0, "fp": 9, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 5, "fp": 3, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 5, "fp": 7, "fn": 1}, "core": {"tp": 5, "fp": 7, "fn": 1}, "all": {"tp": 5, "fp": 7, "fn": 1}}, "coderabbit": {"strict": {"tp": 4, "fp": 6, "fn": 2}, "core": {"tp": 4, "fp": 6, "fn": 2}, "all": {"tp": 4, "fp": 6, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 8, "fn": 3}, "core": {"tp": 3, "fp": 8, "fn": 3}, "all": {"tp": 3, "fp": 8, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 4, "fp": 4, "fn": 2}, "core": {"tp": 4, "fp": 4, "fn": 2}, "all": {"tp": 4, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 1, "fp": 6, "fn": 5}, "core": {"tp": 1, "fp": 6, "fn": 5}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 6}, "core": {"tp": 2, "fp": 2, "fn": 6}, "all": {"tp": 2, "fp": 2, "fn": 6}}, "codeant-v2": {"strict": {"tp": 1, "fp": 17, "fn": 7}, "core": {"tp": 1, "fp": 17, "fn": 7}, "all": {"tp": 1, "fp": 17, "fn": 7}}, "coderabbit": {"strict": {"tp": 4, "fp": 23, "fn": 4}, "core": {"tp": 4, "fp": 23, "fn": 4}, "all": {"tp": 4, "fp": 23, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 3, "fn": 5}, "core": {"tp": 3, "fp": 3, "fn": 5}, "all": {"tp": 3, "fp": 3, "fn": 5}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 4, "fp": 5, "fn": 4}, "core": {"tp": 4, "fp": 5, "fn": 4}, "all": {"tp": 4, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 5, "fp": 6, "fn": 3}, "core": {"tp": 5, "fp": 6, "fn": 3}, "all": {"tp": 5, "fp": 6, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "bugbot": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}, "claude": {"strict": {"tp": 3, "fp": 3, "fn": 5}, "core": {"tp": 3, "fp": 3, "fn": 5}, "all": {"tp": 3, "fp": 3, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 7}, "core": {"tp": 1, "fp": 3, "fn": 7}, "all": {"tp": 1, "fp": 3, "fn": 7}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kodus-v2": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 5, "fn": 7}, "core": {"tp": 1, "fp": 5, "fn": 7}, "all": {"tp": 1, "fp": 5, "fn": 7}}, "qodo-v2": {"strict": {"tp": 5, "fp": 13, "fn": 3}, "core": {"tp": 5, "fp": 13, "fn": 3}, "all": {"tp": 5, "fp": 13, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "codeant-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "copilot-v2": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 5, "fn": 0}, "core": {"tp": 2, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 4, "fp": 1, "fn": 5}, "core": {"tp": 4, "fp": 1, "fn": 5}, "all": {"tp": 4, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 4, "fp": 5, "fn": 5}, "core": {"tp": 4, "fp": 5, "fn": 5}, "all": {"tp": 4, "fp": 5, "fn": 5}}, "coderabbit": {"strict": {"tp": 8, "fp": 13, "fn": 1}, "core": {"tp": 8, "fp": 13, "fn": 1}, "all": {"tp": 8, "fp": 13, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 7, "fp": 2, "fn": 2}, "core": {"tp": 7, "fp": 2, "fn": 2}, "all": {"tp": 7, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 4, "fn": 4}, "core": {"tp": 5, "fp": 4, "fn": 4}, "all": {"tp": 5, "fp": 4, "fn": 4}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 5}, "core": {"tp": 4, "fp": 2, "fn": 5}, "all": {"tp": 4, "fp": 2, "fn": 5}}, "gitlab": {"strict": {"tp": 5, "fp": 5, "fn": 4}, "core": {"tp": 5, "fp": 5, "fn": 4}, "all": {"tp": 5, "fp": 5, "fn": 4}}, "copilot-v2": {"strict": {"tp": 7, "fp": 3, "fn": 2}, "core": {"tp": 7, "fp": 3, "fn": 2}, "all": {"tp": 7, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 8, "fp": 5, "fn": 1}, "core": {"tp": 8, "fp": 5, "fn": 1}, "all": {"tp": 8, "fp": 5, "fn": 1}}, "bugbot": {"strict": {"tp": 6, "fp": 2, "fn": 3}, "core": {"tp": 6, "fp": 2, "fn": 3}, "all": {"tp": 6, "fp": 2, "fn": 3}}, "claude": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 4, "fp": 0, "fn": 5}, "core": {"tp": 4, "fp": 0, "fn": 5}, "all": {"tp": 4, "fp": 0, "fn": 5}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 6}, "core": {"tp": 3, "fp": 0, "fn": 6}, "all": {"tp": 3, "fp": 0, "fn": 6}}, "devin": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 8}, "core": {"tp": 1, "fp": 1, "fn": 8}, "all": {"tp": 1, "fp": 1, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 9}, "core": {"tp": 0, "fp": 0, "fn": 9}, "all": {"tp": 0, "fp": 0, "fn": 9}}, "kodus-v2": {"strict": {"tp": 4, "fp": 3, "fn": 5}, "core": {"tp": 4, "fp": 3, "fn": 5}, "all": {"tp": 4, "fp": 3, "fn": 5}}, "macroscope": {"strict": {"tp": 5, "fp": 5, "fn": 4}, "core": {"tp": 5, "fp": 5, "fn": 4}, "all": {"tp": 5, "fp": 5, "fn": 4}}, "qodo-v2": {"strict": {"tp": 9, "fp": 2, "fn": 0}, "core": {"tp": 9, "fp": 2, "fn": 0}, "all": {"tp": 9, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 12, "fn": 1}, "core": {"tp": 2, "fp": 12, "fn": 1}, "all": {"tp": 2, "fp": 12, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "coderabbit": {"strict": {"tp": 4, "fp": 7, "fn": 2}, "core": {"tp": 4, "fp": 7, "fn": 2}, "all": {"tp": 4, "fp": 7, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "copilot-v2": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "qodo-v2": {"strict": {"tp": 5, "fp": 3, "fn": 1}, "core": {"tp": 5, "fp": 3, "fn": 1}, "all": {"tp": 5, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 1}, "core": {"tp": 1, "fp": 7, "fn": 3}, "all": {"tp": 2, "fp": 7, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 4, "fp": 3, "fn": 0}, "all": {"tp": 5, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 10, "fn": 2}, "core": {"tp": 1, "fp": 10, "fn": 3}, "all": {"tp": 2, "fp": 10, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 4}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "qodo-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 2}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 6, "fn": 3}, "core": {"tp": 2, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 4}}, "coderabbit": {"strict": {"tp": 5, "fp": 15, "fn": 0}, "core": {"tp": 5, "fp": 15, "fn": 0}, "all": {"tp": 5, "fp": 15, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "gemini-v2": {"strict": {"tp": 1, "fp": 6, "fn": 4}, "core": {"tp": 1, "fp": 6, "fn": 4}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 4, "fp": 8, "fn": 1}, "core": {"tp": 4, "fp": 8, "fn": 1}, "all": {"tp": 4, "fp": 8, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 3, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "claude-code": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 1, "fp": 7, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}], "tools": ["augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"greptile-v4-1": {"strict": {"precision": 40.2, "recall": 50.4, "f1": 44.7, "f2": 47.9, "tp": 70, "fp": 104, "fn": 69, "num_prs": 50}, "core": {"precision": 42.2, "recall": 48.1, "f1": 45.0, "f2": 46.8, "tp": 76, "fp": 104, "fn": 82, "num_prs": 50}, "all": {"precision": 44.7, "recall": 48.6, "f1": 46.5, "f2": 47.7, "tp": 84, "fp": 104, "fn": 89, "num_prs": 50}}, "codeant-v2": {"strict": {"precision": 29.3, "recall": 38.1, "f1": 33.1, "f2": 36.0, "tp": 53, "fp": 128, "fn": 86, "num_prs": 50}, "core": {"precision": 30.4, "recall": 35.4, "f1": 32.7, "f2": 34.3, "tp": 56, "fp": 128, "fn": 102, "num_prs": 50}, "all": {"precision": 31.2, "recall": 33.5, "f1": 32.3, "f2": 33.0, "tp": 58, "fp": 128, "fn": 115, "num_prs": 50}}, "baz": {"strict": {"precision": 43.9, "recall": 36.0, "f1": 39.5, "f2": 37.3, "tp": 50, "fp": 64, "fn": 89, "num_prs": 50}, "core": {"precision": 44.8, "recall": 32.9, "f1": 38.0, "f2": 34.8, "tp": 52, "fp": 64, "fn": 106, "num_prs": 50}, "all": {"precision": 46.2, "recall": 31.8, "f1": 37.7, "f2": 33.9, "tp": 55, "fp": 64, "fn": 118, "num_prs": 50}}, "claude-code": {"strict": {"precision": 38.2, "recall": 43.2, "f1": 40.5, "f2": 42.1, "tp": 60, "fp": 97, "fn": 79, "num_prs": 50}, "core": {"precision": 40.1, "recall": 41.1, "f1": 40.6, "f2": 40.9, "tp": 65, "fp": 97, "fn": 93, "num_prs": 50}, "all": {"precision": 42.3, "recall": 41.0, "f1": 41.6, "f2": 41.3, "tp": 71, "fp": 97, "fn": 102, "num_prs": 50}}, "kg": {"strict": {"precision": 53.5, "recall": 16.5, "f1": 25.3, "f2": 19.2, "tp": 23, "fp": 20, "fn": 116, "num_prs": 50}, "core": {"precision": 54.5, "recall": 15.2, "f1": 23.8, "f2": 17.8, "tp": 24, "fp": 20, "fn": 134, "num_prs": 50}, "all": {"precision": 55.6, "recall": 14.5, "f1": 22.9, "f2": 17.0, "tp": 25, "fp": 20, "fn": 148, "num_prs": 50}}, "graphite": {"strict": {"precision": 71.4, "recall": 7.2, "f1": 13.1, "f2": 8.8, "tp": 10, "fp": 4, "fn": 129, "num_prs": 50}, "core": {"precision": 73.3, "recall": 7.0, "f1": 12.7, "f2": 8.5, "tp": 11, "fp": 4, "fn": 147, "num_prs": 50}, "all": {"precision": 73.3, "recall": 6.4, "f1": 11.7, "f2": 7.8, "tp": 11, "fp": 4, "fn": 162, "num_prs": 50}}, "bugbot": {"strict": {"precision": 49.6, "recall": 47.5, "f1": 48.5, "f2": 47.9, "tp": 66, "fp": 67, "fn": 73, "num_prs": 50}, "core": {"precision": 51.8, "recall": 45.6, "f1": 48.5, "f2": 46.7, "tp": 72, "fp": 67, "fn": 86, "num_prs": 50}, "all": {"precision": 52.8, "recall": 43.4, "f1": 47.6, "f2": 45.0, "tp": 75, "fp": 67, "fn": 98, "num_prs": 50}}, "devin": {"strict": {"precision": 56.6, "recall": 43.2, "f1": 49.0, "f2": 45.3, "tp": 60, "fp": 46, "fn": 79, "num_prs": 50}, "core": {"precision": 58.6, "recall": 41.1, "f1": 48.3, "f2": 43.7, "tp": 65, "fp": 46, "fn": 93, "num_prs": 50}, "all": {"precision": 59.3, "recall": 38.7, "f1": 46.9, "f2": 41.6, "tp": 67, "fp": 46, "fn": 106, "num_prs": 50}}, "gitlab": {"strict": {"precision": 42.2, "recall": 48.9, "f1": 45.3, "f2": 47.4, "tp": 68, "fp": 93, "fn": 71, "num_prs": 50}, "core": {"precision": 45.0, "recall": 48.1, "f1": 46.5, "f2": 47.4, "tp": 76, "fp": 93, "fn": 82, "num_prs": 50}, "all": {"precision": 46.9, "recall": 47.4, "f1": 47.1, "f2": 47.3, "tp": 82, "fp": 93, "fn": 91, "num_prs": 50}}, "claude": {"strict": {"precision": 36.7, "recall": 38.8, "f1": 37.8, "f2": 38.4, "tp": 54, "fp": 93, "fn": 85, "num_prs": 50}, "core": {"precision": 38.8, "recall": 37.3, "f1": 38.1, "f2": 37.6, "tp": 59, "fp": 93, "fn": 99, "num_prs": 50}, "all": {"precision": 41.1, "recall": 37.6, "f1": 39.3, "f2": 38.2, "tp": 65, "fp": 93, "fn": 108, "num_prs": 50}}, "copilot-v2": {"strict": {"precision": 30.4, "recall": 63.3, "f1": 41.1, "f2": 52.1, "tp": 88, "fp": 201, "fn": 51, "num_prs": 50}, "core": {"precision": 33.2, "recall": 63.3, "f1": 43.6, "f2": 53.6, "tp": 100, "fp": 201, "fn": 58, "num_prs": 50}, "all": {"precision": 35.2, "recall": 63.0, "f1": 45.1, "f2": 54.4, "tp": 109, "fp": 201, "fn": 64, "num_prs": 50}}, "qodo-v2": {"strict": {"precision": 44.1, "recall": 64.7, "f1": 52.5, "f2": 59.2, "tp": 90, "fp": 114, "fn": 49, "num_prs": 50}, "core": {"precision": 46.5, "recall": 62.7, "f1": 53.4, "f2": 58.6, "tp": 99, "fp": 114, "fn": 59, "num_prs": 50}, "all": {"precision": 47.5, "recall": 59.5, "f1": 52.8, "f2": 56.7, "tp": 103, "fp": 114, "fn": 70, "num_prs": 50}}, "macroscope": {"strict": {"precision": 47.7, "recall": 44.6, "f1": 46.1, "f2": 45.2, "tp": 62, "fp": 68, "fn": 77, "num_prs": 50}, "core": {"precision": 48.9, "recall": 41.1, "f1": 44.7, "f2": 42.5, "tp": 65, "fp": 68, "fn": 93, "num_prs": 50}, "all": {"precision": 50.0, "recall": 39.3, "f1": 44.0, "f2": 41.1, "tp": 68, "fp": 68, "fn": 105, "num_prs": 50}}, "qodo-extended-v2": {"strict": {"precision": 55.9, "recall": 61.2, "f1": 58.4, "f2": 60.0, "tp": 85, "fp": 67, "fn": 54, "num_prs": 50}, "core": {"precision": 58.4, "recall": 59.5, "f1": 58.9, "f2": 59.3, "tp": 94, "fp": 67, "fn": 64, "num_prs": 50}, "all": {"precision": 58.9, "recall": 55.5, "f1": 57.1, "f2": 56.1, "tp": 96, "fp": 67, "fn": 77, "num_prs": 50}}, "cubic-v2": {"strict": {"precision": 53.2, "recall": 59.7, "f1": 56.3, "f2": 58.3, "tp": 83, "fp": 73, "fn": 56, "num_prs": 50}, "core": {"precision": 56.8, "recall": 60.8, "f1": 58.7, "f2": 59.9, "tp": 96, "fp": 73, "fn": 62, "num_prs": 50}, "all": {"precision": 57.8, "recall": 57.8, "f1": 57.8, "f2": 57.8, "tp": 100, "fp": 73, "fn": 73, "num_prs": 50}}, "kodus-v2": {"strict": {"precision": 50.5, "recall": 36.0, "f1": 42.0, "f2": 38.2, "tp": 50, "fp": 49, "fn": 89, "num_prs": 50}, "core": {"precision": 52.4, "recall": 34.2, "f1": 41.4, "f2": 36.7, "tp": 54, "fp": 49, "fn": 104, "num_prs": 50}, "all": {"precision": 53.8, "recall": 32.9, "f1": 40.9, "f2": 35.7, "tp": 57, "fp": 49, "fn": 116, "num_prs": 50}}, "gemini-v2": {"strict": {"precision": 36.2, "recall": 54.0, "f1": 43.4, "f2": 49.1, "tp": 75, "fp": 132, "fn": 64, "num_prs": 50}, "core": {"precision": 38.0, "recall": 51.3, "f1": 43.7, "f2": 47.9, "tp": 81, "fp": 132, "fn": 77, "num_prs": 50}, "all": {"precision": 40.3, "recall": 51.4, "f1": 45.2, "f2": 48.7, "tp": 89, "fp": 132, "fn": 84, "num_prs": 50}}, "sourcery": {"strict": {"precision": 33.3, "recall": 50.4, "f1": 40.1, "f2": 45.7, "tp": 70, "fp": 140, "fn": 69, "num_prs": 50}, "core": {"precision": 36.1, "recall": 50.0, "f1": 41.9, "f2": 46.4, "tp": 79, "fp": 140, "fn": 79, "num_prs": 50}, "all": {"precision": 38.1, "recall": 49.7, "f1": 43.1, "f2": 46.8, "tp": 86, "fp": 140, "fn": 87, "num_prs": 50}}, "augment": {"strict": {"precision": 45.9, "recall": 59.7, "f1": 51.9, "f2": 56.3, "tp": 83, "fp": 98, "fn": 56, "num_prs": 50}, "core": {"precision": 49.5, "recall": 60.8, "f1": 54.5, "f2": 58.1, "tp": 96, "fp": 98, "fn": 62, "num_prs": 50}, "all": {"precision": 50.8, "recall": 58.4, "f1": 54.3, "f2": 56.7, "tp": 101, "fp": 98, "fn": 72, "num_prs": 50}}, "coderabbit": {"strict": {"precision": 26.8, "recall": 61.9, "f1": 37.4, "f2": 49.0, "tp": 86, "fp": 235, "fn": 53, "num_prs": 50}, "core": {"precision": 28.6, "recall": 59.5, "f1": 38.6, "f2": 48.9, "tp": 94, "fp": 235, "fn": 64, "num_prs": 50}, "all": {"precision": 30.7, "recall": 60.1, "f1": 40.6, "f2": 50.4, "tp": 104, "fp": 235, "fn": 69, "num_prs": 50}}}}};
+        const toolDisplayNames = {"graphite": "Graphite", "qodo": "Qodo", "gemini": "Gemini", "claude": "Claude Code", "augment": "Augment", "bugbot": "Cursor Bugbot", "coderabbit": "CodeRabbit", "propel": "Propel", "copilot": "GitHub Copilot", "baz": "Baz", "greptile": "Greptile", "kg": "KG", "entelligence": "Entelligence", "cubic-v2": "Cubic v2", "sourcery": "Sourcery", "mesa": "Mesa", "codeant-v2": "CodeAnt", "claude-code": "Claude Code (CLI)", "devin": "Devin", "kodus-v2": "Kodus", "greptile-v4-1": "Greptile v4", "gemini-v2": "Gemini", "gitlab": "GitLab Duo Code Review", "qodo-v2": "Qodo v2", "qodo-extended-v2": "Qodo Extended", "macroscope": "Macroscope", "deputydev-v1": "DeputyDev v1"};
+        const toolColors = {"graphite": "#6366f1", "qodo": "#8b5cf6", "gemini": "#06b6d4", "claude": "#f59e0b", "augment": "#10b981", "bugbot": "#3b82f6", "coderabbit": "#ec4899", "propel": "#14b8a6", "propel-v2": "#0d9488", "copilot": "#6b7280", "baz": "#f97316", "greptile": "#22c55e", "kg": "#a855f7", "entelligence": "#0ea5e9", "cubic-dev": "#d946ef", "sourcery": "#84cc16", "mesa": "#f43f5e", "codeant": "#e11d48", "codeant-v2": "#fb7185", "claude-code": "#d97706", "devin": "#7c3aed", "kodus-v2": "#059669", "greptile-v4": "#16a34a", "qodo-v2": "#7c3aed", "qodo-extended-v2": "#6d28d9", "macroscope": "#0891b2", "cubic-v2": "#c026d3", "deputydev-v1": "#2563eb"};
+        const predefinedFilters = [{"id": "high_precision", "label": "Highest Precision", "filters": {}, "sort": "precision", "description": "Tools ranked by precision - fewer false positives, more reliable findings", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "graphite", "best_score": 100.0}, {"id": "tool_kg_language_ruby", "label": "Best for Ruby (Precision)", "filters": {"language": ["Ruby"]}, "description": "Ruby codebases with Rails patterns.", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "kg", "best_score": 100.0, "sort": "precision"}, {"id": "tool_sourcery_difficulty_moderate_context_file", "label": "Moderate Bugs + File Context (Recall)", "filters": {"difficulty": ["moderate"], "context": ["file"]}, "description": "Requires careful reading. Requires full file understanding.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "sourcery", "best_score": 90.0, "sort": "recall"}, {"id": "tool_coderabbit_language_typescript_domain_scheduling", "label": "Typescript + Scheduling (Recall)", "filters": {"language": ["TypeScript"], "domain": ["scheduling"]}, "description": "TypeScript codebases with frontend patterns. Task scheduling.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "coderabbit", "best_score": 88.9, "sort": "recall"}, {"id": "go_small", "label": "Best for Small Go PRs", "filters": {"language": ["Go"], "pr_size": ["small"]}, "description": "Go codebases with concurrency patterns. Small PRs with 1-2 files, easier to review thoroughly", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 88.2}, {"id": "complexity_complex", "label": "Best for Complex Code", "filters": {"complexity": ["complex"]}, "description": "Deep logic and dependencies.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 81.1}, {"id": "domain_ui", "label": "Best for Ui", "filters": {"domain": ["UI"]}, "description": "User interface and frontend.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "bugbot", "best_score": 78.3}, {"id": "risk_critical", "label": "Best for Critical Risk", "filters": {"risk": ["critical"]}, "description": "Critical security or data corruption risk.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-v2", "best_score": 75.9}, {"id": "tool_codeant-v2_pr_size_small_change_type_performance", "label": "Small PRs + Performance Optimization (Precision)", "filters": {"pr_size": ["small"], "change_type": ["performance"]}, "description": "Small PRs with 1-2 files, easier to review thoroughly Performance optimization changes", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "codeant-v2", "best_score": 75.0, "sort": "precision"}, {"id": "tool_gitlab_change_type_bug_fix_context_cross_file", "label": "Bug Fixes + Cross-File", "filters": {"change_type": ["bug_fix"], "context": ["cross_file"]}, "description": "Bug fixes and issue resolution Spans multiple files.", "best_model": "openai_gpt-5.2", "best_tool": "gitlab", "best_score": 74.3}, {"id": "tool_baz_language_java_domain_authentication", "label": "Java + Authentication", "filters": {"language": ["Java"], "domain": ["authentication"]}, "description": "Java codebases with OOP patterns. Auth and access control.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "baz", "best_score": 74.1}, {"id": "tool_deputydev-v1_risk_high_context_file", "label": "High Risk + File Context", "filters": {"risk": ["high"], "context": ["file"]}, "description": "Significant impact, potential data loss. Requires full file understanding.", "best_model": "gpt-5.5", "best_tool": "deputydev-v1", "best_score": 73.7}, {"id": "python_medium", "label": "Best for Medium Python PRs", "filters": {"language": ["Python"], "pr_size": ["medium"]}, "description": "Python codebases with dynamic typing. Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 72.7}, {"id": "domain_concurrency", "label": "Best for Concurrency", "filters": {"domain": ["concurrency"]}, "description": "Threading and async operations.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "macroscope", "best_score": 69.6}, {"id": "tool_claude_change_type_bug_fix", "label": "Best for Bug Fixes", "filters": {"change_type": ["bug_fix"]}, "description": "Bug fixes and issue resolution", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "claude", "best_score": 68.3}, {"id": "tool_greptile-v4-1_language_ruby_pr_size_medium", "label": "Ruby + Medium PRs", "filters": {"language": ["Ruby"], "pr_size": ["medium"]}, "description": "Ruby codebases with Rails patterns. Medium PRs with 3-5 files, typical feature development", "best_model": "openai_gpt-5.2", "best_tool": "greptile-v4-1", "best_score": 66.7}, {"id": "high_recall", "label": "Highest Recall", "filters": {}, "sort": "recall", "description": "Tools ranked by recall - catches more issues, may have more noise", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "copilot-v2", "best_score": 66.5}, {"id": "tool_kodus-v2_language_ruby_change_type_feature", "label": "Ruby + Features", "filters": {"language": ["Ruby"], "change_type": ["feature"]}, "description": "Ruby codebases with Rails patterns. New feature implementation", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "kodus-v2", "best_score": 61.2}, {"id": "difficulty_moderate", "label": "Best for Moderate Bugs", "filters": {"difficulty": ["moderate"]}, "description": "Requires careful reading.", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "devin", "best_score": 57.9}, {"id": "tool_gemini-v2_language_ruby_difficulty_subtle", "label": "Ruby + Subtle Bugs", "filters": {"language": ["Ruby"], "difficulty": ["subtle"]}, "description": "Ruby codebases with Rails patterns. Non-obvious, needs domain knowledge.", "best_model": "openai_gpt-5.2", "best_tool": "gemini-v2", "best_score": 56.4}, {"id": "language_go", "label": "Best for Go", "filters": {"language": ["Go"]}, "description": "Go codebases with concurrency patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 84.2}, {"id": "change_type_performance", "label": "Best for Performance Optimization", "filters": {"change_type": ["performance"]}, "description": "Performance optimization changes", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 82.9}, {"id": "complex_subtle", "label": "Complex & Subtle", "filters": {"complexity": ["complex"], "difficulty": ["subtle", "very_subtle"]}, "description": "Deep logic and dependencies. Non-obvious, needs domain knowledge.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 81.1}, {"id": "pr_size_small", "label": "Best for Small PRs", "filters": {"pr_size": ["small"]}, "description": "Small PRs with 1-2 files, easier to review thoroughly", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 79.6}, {"id": "domain_caching", "label": "Best for Caching", "filters": {"domain": ["caching"]}, "description": "Cache and memoization.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 78.6}, {"id": "pr_size_large", "label": "Best for Large PRs", "filters": {"pr_size": ["large"]}, "description": "Large PRs with 6+ files, complex changes requiring careful review", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 74.5}, {"id": "change_type_bug_fix", "label": "Best for Bug Fixes", "filters": {"change_type": ["bug_fix"]}, "description": "Bug fixes and issue resolution", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 73.2}, {"id": "language_python", "label": "Best for Python", "filters": {"language": ["Python"]}, "description": "Python codebases with dynamic typing.", "best_model": "openai_gpt-5.2", "best_tool": "cubic-v2", "best_score": 72.5}, {"id": "high_risk_auth", "label": "High Risk Auth", "filters": {"risk": ["high", "critical"], "domain": ["authentication"]}, "description": "Significant impact, potential data loss. Auth and access control.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 71.9}, {"id": "context_file", "label": "Best for File Context", "filters": {"context": ["file"]}, "description": "Requires full file understanding.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 71.6}, {"id": "risk_high", "label": "Best for High Risk", "filters": {"risk": ["high"]}, "description": "Significant impact, potential data loss.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 71.3}, {"id": "language_java", "label": "Best for Java", "filters": {"language": ["Java"]}, "description": "Java codebases with OOP patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 70.8}, {"id": "domain_authentication", "label": "Best for Authentication", "filters": {"domain": ["authentication"]}, "description": "Auth and access control.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 69.2}, {"id": "ruby_medium", "label": "Best for Medium Ruby PRs", "filters": {"language": ["Ruby"], "pr_size": ["medium"]}, "description": "Ruby codebases with Rails patterns. Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 68.8}, {"id": "java_medium", "label": "Best for Medium Java PRs", "filters": {"language": ["Java"], "pr_size": ["medium"]}, "description": "Java codebases with OOP patterns. Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 68.6}, {"id": "domain_scheduling", "label": "Best for Scheduling", "filters": {"domain": ["scheduling"]}, "description": "Task scheduling.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 68.3}, {"id": "language_typescript", "label": "Best for Typescript", "filters": {"language": ["TypeScript"]}, "description": "TypeScript codebases with frontend patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 68.2}, {"id": "context_cross_file", "label": "Best for Cross-File", "filters": {"context": ["cross_file"]}, "description": "Spans multiple files.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 67.5}, {"id": "pr_size_medium", "label": "Best for Medium PRs", "filters": {"pr_size": ["medium"]}, "description": "Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 67.1}, {"id": "concern_reliability", "label": "Best for Reliability", "filters": {"concern": ["reliability"]}, "description": "Error handling and system stability", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-v2", "best_score": 66.7}, {"id": "difficulty_subtle", "label": "Best for Subtle Bugs", "filters": {"difficulty": ["subtle"]}, "description": "Non-obvious, needs domain knowledge.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 66.4}, {"id": "concern_correctness", "label": "Best for Correctness", "filters": {"concern": ["correctness"]}, "description": "Logical correctness and expected behavior", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 66.4}, {"id": "high_f1", "label": "Highest Score", "filters": {}, "sort": "f1", "description": "Tools ranked by F-beta score (default F2, recall-weighted)", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 65.8}, {"id": "concern_security", "label": "Best for Security", "filters": {"concern": ["security"]}, "description": "Security vulnerabilities and attack vectors", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 65.2}, {"id": "security_critical", "label": "Security Critical", "filters": {"concern": ["security"], "risk": ["high", "critical"]}, "description": "Security vulnerabilities and attack vectors Significant impact, potential data loss.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 65.2}, {"id": "language_ruby", "label": "Best for Ruby", "filters": {"language": ["Ruby"]}, "description": "Ruby codebases with Rails patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 63.8}, {"id": "complexity_moderate", "label": "Best for Moderate Code", "filters": {"complexity": ["moderate"]}, "description": "Some abstraction.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 63.4}, {"id": "change_type_feature", "label": "Best for Features", "filters": {"change_type": ["feature"]}, "description": "New feature implementation", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 62.9}, {"id": "risk_medium", "label": "Best for Medium Risk", "filters": {"risk": ["medium"]}, "description": "Moderate user impact.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 59.5}];
 
         let currentModel = 'anthropic_claude-opus-4-5-20251101';
         let currentProfile = 'core';
diff --git a/offline/analysis/benchmark_dashboard.json b/offline/analysis/benchmark_dashboard.json
index 95e8ac7..56a9873 100644
--- a/offline/analysis/benchmark_dashboard.json
+++ b/offline/analysis/benchmark_dashboard.json
@@ -17842,102 +17842,6 @@
         ]
       },
       "overall_metrics": {
-        "augment": {
-          "strict": {
-            "precision": 56.5,
-            "recall": 65.5,
-            "f1": 60.7,
-            "f2": 63.5,
-            "tp": 91,
-            "fp": 70,
-            "fn": 48,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 59.5,
-            "recall": 65.2,
-            "f1": 62.2,
-            "f2": 64.0,
-            "tp": 103,
-            "fp": 70,
-            "fn": 55,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 60.7,
-            "recall": 62.4,
-            "f1": 61.5,
-            "f2": 62.1,
-            "tp": 108,
-            "fp": 70,
-            "fn": 65,
-            "num_prs": 50
-          }
-        },
-        "cubic-v2": {
-          "strict": {
-            "precision": 58.3,
-            "recall": 65.5,
-            "f1": 61.7,
-            "f2": 63.9,
-            "tp": 91,
-            "fp": 65,
-            "fn": 48,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 61.5,
-            "recall": 65.8,
-            "f1": 63.6,
-            "f2": 64.9,
-            "tp": 104,
-            "fp": 65,
-            "fn": 54,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 62.4,
-            "recall": 62.4,
-            "f1": 62.4,
-            "f2": 62.4,
-            "tp": 108,
-            "fp": 65,
-            "fn": 65,
-            "num_prs": 50
-          }
-        },
-        "copilot-v2": {
-          "strict": {
-            "precision": 35.6,
-            "recall": 66.9,
-            "f1": 46.5,
-            "f2": 56.9,
-            "tp": 93,
-            "fp": 168,
-            "fn": 46,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 38.5,
-            "recall": 66.5,
-            "f1": 48.7,
-            "f2": 58.0,
-            "tp": 105,
-            "fp": 168,
-            "fn": 53,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 40.6,
-            "recall": 66.5,
-            "f1": 50.4,
-            "f2": 59.0,
-            "tp": 115,
-            "fp": 168,
-            "fn": 58,
-            "num_prs": 50
-          }
-        },
         "greptile-v4-1": {
           "strict": {
             "precision": 49.0,
@@ -17970,102 +17874,6 @@
             "num_prs": 50
           }
         },
-        "sourcery": {
-          "strict": {
-            "precision": 38.4,
-            "recall": 56.1,
-            "f1": 45.6,
-            "f2": 51.4,
-            "tp": 78,
-            "fp": 125,
-            "fn": 61,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 41.0,
-            "recall": 55.1,
-            "f1": 47.0,
-            "f2": 51.5,
-            "tp": 87,
-            "fp": 125,
-            "fn": 71,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 43.2,
-            "recall": 54.9,
-            "f1": 48.3,
-            "f2": 52.1,
-            "tp": 95,
-            "fp": 125,
-            "fn": 78,
-            "num_prs": 50
-          }
-        },
-        "qodo-v2": {
-          "strict": {
-            "precision": 53.0,
-            "recall": 64.0,
-            "f1": 58.0,
-            "f2": 61.5,
-            "tp": 89,
-            "fp": 79,
-            "fn": 50,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 55.4,
-            "recall": 62.0,
-            "f1": 58.5,
-            "f2": 60.6,
-            "tp": 98,
-            "fp": 79,
-            "fn": 60,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 56.4,
-            "recall": 59.0,
-            "f1": 57.6,
-            "f2": 58.4,
-            "tp": 102,
-            "fp": 79,
-            "fn": 71,
-            "num_prs": 50
-          }
-        },
-        "gitlab": {
-          "strict": {
-            "precision": 51.0,
-            "recall": 54.0,
-            "f1": 52.4,
-            "f2": 53.3,
-            "tp": 75,
-            "fp": 72,
-            "fn": 64,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 53.5,
-            "recall": 52.5,
-            "f1": 53.0,
-            "f2": 52.7,
-            "tp": 83,
-            "fp": 72,
-            "fn": 75,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 55.8,
-            "recall": 52.6,
-            "f1": 54.2,
-            "f2": 53.2,
-            "tp": 91,
-            "fp": 72,
-            "fn": 82,
-            "num_prs": 50
-          }
-        },
         "codeant-v2": {
           "strict": {
             "precision": 39.0,
@@ -18098,35 +17906,35 @@
             "num_prs": 50
           }
         },
-        "bugbot": {
+        "baz": {
           "strict": {
-            "precision": 54.8,
-            "recall": 48.9,
-            "f1": 51.7,
-            "f2": 50.0,
-            "tp": 68,
-            "fp": 56,
-            "fn": 71,
+            "precision": 60.6,
+            "recall": 41.0,
+            "f1": 48.9,
+            "f2": 43.8,
+            "tp": 57,
+            "fp": 37,
+            "fn": 82,
             "num_prs": 50
           },
           "core": {
-            "precision": 56.9,
-            "recall": 46.8,
-            "f1": 51.4,
-            "f2": 48.6,
-            "tp": 74,
-            "fp": 56,
-            "fn": 84,
+            "precision": 61.5,
+            "recall": 37.3,
+            "f1": 46.5,
+            "f2": 40.5,
+            "tp": 59,
+            "fp": 37,
+            "fn": 99,
             "num_prs": 50
           },
           "all": {
-            "precision": 58.2,
-            "recall": 45.1,
-            "f1": 50.8,
-            "f2": 47.2,
-            "tp": 78,
-            "fp": 56,
-            "fn": 95,
+            "precision": 62.6,
+            "recall": 35.8,
+            "f1": 45.6,
+            "f2": 39.2,
+            "tp": 62,
+            "fp": 37,
+            "fn": 111,
             "num_prs": 50
           }
         },
@@ -18162,6 +17970,102 @@
             "num_prs": 50
           }
         },
+        "kg": {
+          "strict": {
+            "precision": 53.3,
+            "recall": 17.3,
+            "f1": 26.1,
+            "f2": 20.0,
+            "tp": 24,
+            "fp": 21,
+            "fn": 115,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 54.3,
+            "recall": 15.8,
+            "f1": 24.5,
+            "f2": 18.4,
+            "tp": 25,
+            "fp": 21,
+            "fn": 133,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 55.3,
+            "recall": 15.0,
+            "f1": 23.6,
+            "f2": 17.6,
+            "tp": 26,
+            "fp": 21,
+            "fn": 147,
+            "num_prs": 50
+          }
+        },
+        "graphite": {
+          "strict": {
+            "precision": 100.0,
+            "recall": 7.9,
+            "f1": 14.7,
+            "f2": 9.7,
+            "tp": 11,
+            "fp": 0,
+            "fn": 128,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 100.0,
+            "recall": 7.6,
+            "f1": 14.1,
+            "f2": 9.3,
+            "tp": 12,
+            "fp": 0,
+            "fn": 146,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 100.0,
+            "recall": 6.9,
+            "f1": 13.0,
+            "f2": 8.5,
+            "tp": 12,
+            "fp": 0,
+            "fn": 161,
+            "num_prs": 50
+          }
+        },
+        "bugbot": {
+          "strict": {
+            "precision": 54.8,
+            "recall": 48.9,
+            "f1": 51.7,
+            "f2": 50.0,
+            "tp": 68,
+            "fp": 56,
+            "fn": 71,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 56.9,
+            "recall": 46.8,
+            "f1": 51.4,
+            "f2": 48.6,
+            "tp": 74,
+            "fp": 56,
+            "fn": 84,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 58.2,
+            "recall": 45.1,
+            "f1": 50.8,
+            "f2": 47.2,
+            "tp": 78,
+            "fp": 56,
+            "fn": 95,
+            "num_prs": 50
+          }
+        },
         "devin": {
           "strict": {
             "precision": 67.4,
@@ -18194,35 +18098,35 @@
             "num_prs": 50
           }
         },
-        "macroscope": {
+        "gitlab": {
           "strict": {
-            "precision": 60.2,
-            "recall": 53.2,
-            "f1": 56.5,
-            "f2": 54.5,
-            "tp": 74,
-            "fp": 49,
-            "fn": 65,
+            "precision": 51.0,
+            "recall": 54.0,
+            "f1": 52.4,
+            "f2": 53.3,
+            "tp": 75,
+            "fp": 72,
+            "fn": 64,
             "num_prs": 50
           },
           "core": {
-            "precision": 61.1,
-            "recall": 48.7,
-            "f1": 54.2,
-            "f2": 50.8,
-            "tp": 77,
-            "fp": 49,
-            "fn": 81,
+            "precision": 53.5,
+            "recall": 52.5,
+            "f1": 53.0,
+            "f2": 52.7,
+            "tp": 83,
+            "fp": 72,
+            "fn": 75,
             "num_prs": 50
           },
           "all": {
-            "precision": 62.3,
-            "recall": 46.8,
-            "f1": 53.5,
-            "f2": 49.3,
-            "tp": 81,
-            "fp": 49,
-            "fn": 92,
+            "precision": 55.8,
+            "recall": 52.6,
+            "f1": 54.2,
+            "f2": 53.2,
+            "tp": 91,
+            "fp": 72,
+            "fn": 82,
             "num_prs": 50
           }
         },
@@ -18258,135 +18162,103 @@
             "num_prs": 50
           }
         },
-        "gemini-v2": {
+        "copilot-v2": {
           "strict": {
-            "precision": 43.2,
-            "recall": 57.6,
-            "f1": 49.4,
-            "f2": 54.0,
-            "tp": 80,
-            "fp": 105,
-            "fn": 59,
+            "precision": 35.6,
+            "recall": 66.9,
+            "f1": 46.5,
+            "f2": 56.9,
+            "tp": 93,
+            "fp": 168,
+            "fn": 46,
             "num_prs": 50
           },
           "core": {
-            "precision": 45.3,
-            "recall": 55.1,
-            "f1": 49.7,
-            "f2": 52.8,
-            "tp": 87,
-            "fp": 105,
-            "fn": 71,
+            "precision": 38.5,
+            "recall": 66.5,
+            "f1": 48.7,
+            "f2": 58.0,
+            "tp": 105,
+            "fp": 168,
+            "fn": 53,
             "num_prs": 50
           },
           "all": {
-            "precision": 47.5,
-            "recall": 54.9,
-            "f1": 50.9,
-            "f2": 53.3,
-            "tp": 95,
-            "fp": 105,
-            "fn": 78,
+            "precision": 40.6,
+            "recall": 66.5,
+            "f1": 50.4,
+            "f2": 59.0,
+            "tp": 115,
+            "fp": 168,
+            "fn": 58,
             "num_prs": 50
           }
         },
-        "coderabbit": {
+        "qodo-v2": {
           "strict": {
-            "precision": 31.0,
-            "recall": 62.6,
-            "f1": 41.4,
-            "f2": 52.0,
-            "tp": 87,
-            "fp": 194,
-            "fn": 52,
+            "precision": 53.0,
+            "recall": 64.0,
+            "f1": 58.0,
+            "f2": 61.5,
+            "tp": 89,
+            "fp": 79,
+            "fn": 50,
             "num_prs": 50
           },
           "core": {
-            "precision": 32.6,
-            "recall": 59.5,
-            "f1": 42.2,
-            "f2": 51.1,
-            "tp": 94,
-            "fp": 194,
-            "fn": 64,
+            "precision": 55.4,
+            "recall": 62.0,
+            "f1": 58.5,
+            "f2": 60.6,
+            "tp": 98,
+            "fp": 79,
+            "fn": 60,
             "num_prs": 50
           },
           "all": {
-            "precision": 34.7,
-            "recall": 59.5,
-            "f1": 43.8,
-            "f2": 52.1,
-            "tp": 103,
-            "fp": 194,
-            "fn": 70,
+            "precision": 56.4,
+            "recall": 59.0,
+            "f1": 57.6,
+            "f2": 58.4,
+            "tp": 102,
+            "fp": 79,
+            "fn": 71,
             "num_prs": 50
           }
         },
-        "baz": {
+        "macroscope": {
           "strict": {
-            "precision": 60.6,
-            "recall": 41.0,
-            "f1": 48.9,
-            "f2": 43.8,
-            "tp": 57,
-            "fp": 37,
-            "fn": 82,
+            "precision": 60.2,
+            "recall": 53.2,
+            "f1": 56.5,
+            "f2": 54.5,
+            "tp": 74,
+            "fp": 49,
+            "fn": 65,
             "num_prs": 50
           },
           "core": {
-            "precision": 61.5,
-            "recall": 37.3,
-            "f1": 46.5,
-            "f2": 40.5,
-            "tp": 59,
-            "fp": 37,
-            "fn": 99,
+            "precision": 61.1,
+            "recall": 48.7,
+            "f1": 54.2,
+            "f2": 50.8,
+            "tp": 77,
+            "fp": 49,
+            "fn": 81,
             "num_prs": 50
           },
           "all": {
-            "precision": 62.6,
-            "recall": 35.8,
-            "f1": 45.6,
-            "f2": 39.2,
-            "tp": 62,
-            "fp": 37,
-            "fn": 111,
+            "precision": 62.3,
+            "recall": 46.8,
+            "f1": 53.5,
+            "f2": 49.3,
+            "tp": 81,
+            "fp": 49,
+            "fn": 92,
             "num_prs": 50
           }
         },
-        "kg": {
-          "strict": {
-            "precision": 53.3,
-            "recall": 17.3,
-            "f1": 26.1,
-            "f2": 20.0,
-            "tp": 24,
-            "fp": 21,
-            "fn": 115,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 54.3,
-            "recall": 15.8,
-            "f1": 24.5,
-            "f2": 18.4,
-            "tp": 25,
-            "fp": 21,
-            "fn": 133,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 55.3,
-            "recall": 15.0,
-            "f1": 23.6,
-            "f2": 17.6,
-            "tp": 26,
-            "fp": 21,
-            "fn": 147,
-            "num_prs": 50
-          }
-        },
-        "qodo-extended-v2": {
+        "qodo-extended-v2": {
           "strict": {
             "precision": 64.8,
             "recall": 66.2,
@@ -18418,6 +18290,38 @@
             "num_prs": 50
           }
         },
+        "cubic-v2": {
+          "strict": {
+            "precision": 58.3,
+            "recall": 65.5,
+            "f1": 61.7,
+            "f2": 63.9,
+            "tp": 91,
+            "fp": 65,
+            "fn": 48,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 61.5,
+            "recall": 65.8,
+            "f1": 63.6,
+            "f2": 64.9,
+            "tp": 104,
+            "fp": 65,
+            "fn": 54,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 62.4,
+            "recall": 62.4,
+            "f1": 62.4,
+            "f2": 62.4,
+            "tp": 108,
+            "fp": 65,
+            "fn": 65,
+            "num_prs": 50
+          }
+        },
         "kodus-v2": {
           "strict": {
             "precision": 57.1,
@@ -18450,35 +18354,131 @@
             "num_prs": 50
           }
         },
-        "graphite": {
+        "gemini-v2": {
           "strict": {
-            "precision": 100.0,
-            "recall": 7.9,
-            "f1": 14.7,
-            "f2": 9.7,
-            "tp": 11,
-            "fp": 0,
-            "fn": 128,
+            "precision": 43.2,
+            "recall": 57.6,
+            "f1": 49.4,
+            "f2": 54.0,
+            "tp": 80,
+            "fp": 105,
+            "fn": 59,
             "num_prs": 50
           },
           "core": {
-            "precision": 100.0,
-            "recall": 7.6,
-            "f1": 14.1,
-            "f2": 9.3,
-            "tp": 12,
-            "fp": 0,
-            "fn": 146,
+            "precision": 45.3,
+            "recall": 55.1,
+            "f1": 49.7,
+            "f2": 52.8,
+            "tp": 87,
+            "fp": 105,
+            "fn": 71,
             "num_prs": 50
           },
           "all": {
-            "precision": 100.0,
-            "recall": 6.9,
-            "f1": 13.0,
-            "f2": 8.5,
-            "tp": 12,
-            "fp": 0,
-            "fn": 161,
+            "precision": 47.5,
+            "recall": 54.9,
+            "f1": 50.9,
+            "f2": 53.3,
+            "tp": 95,
+            "fp": 105,
+            "fn": 78,
+            "num_prs": 50
+          }
+        },
+        "sourcery": {
+          "strict": {
+            "precision": 38.4,
+            "recall": 56.1,
+            "f1": 45.6,
+            "f2": 51.4,
+            "tp": 78,
+            "fp": 125,
+            "fn": 61,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 41.0,
+            "recall": 55.1,
+            "f1": 47.0,
+            "f2": 51.5,
+            "tp": 87,
+            "fp": 125,
+            "fn": 71,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 43.2,
+            "recall": 54.9,
+            "f1": 48.3,
+            "f2": 52.1,
+            "tp": 95,
+            "fp": 125,
+            "fn": 78,
+            "num_prs": 50
+          }
+        },
+        "augment": {
+          "strict": {
+            "precision": 56.5,
+            "recall": 65.5,
+            "f1": 60.7,
+            "f2": 63.5,
+            "tp": 91,
+            "fp": 70,
+            "fn": 48,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 59.5,
+            "recall": 65.2,
+            "f1": 62.2,
+            "f2": 64.0,
+            "tp": 103,
+            "fp": 70,
+            "fn": 55,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 60.7,
+            "recall": 62.4,
+            "f1": 61.5,
+            "f2": 62.1,
+            "tp": 108,
+            "fp": 70,
+            "fn": 65,
+            "num_prs": 50
+          }
+        },
+        "coderabbit": {
+          "strict": {
+            "precision": 31.0,
+            "recall": 62.6,
+            "f1": 41.4,
+            "f2": 52.0,
+            "tp": 87,
+            "fp": 194,
+            "fn": 52,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 32.6,
+            "recall": 59.5,
+            "f1": 42.2,
+            "f2": 51.1,
+            "tp": 94,
+            "fp": 194,
+            "fn": 64,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 34.7,
+            "recall": 59.5,
+            "f1": 43.8,
+            "f2": 52.1,
+            "tp": 103,
+            "fp": 194,
+            "fn": 70,
             "num_prs": 50
           }
         }
@@ -36326,102 +36326,6 @@
         ]
       },
       "overall_metrics": {
-        "augment": {
-          "strict": {
-            "precision": 49.4,
-            "recall": 61.9,
-            "f1": 55.0,
-            "f2": 58.9,
-            "tp": 86,
-            "fp": 88,
-            "fn": 53,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 52.4,
-            "recall": 61.4,
-            "f1": 56.6,
-            "f2": 59.4,
-            "tp": 97,
-            "fp": 88,
-            "fn": 61,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 53.7,
-            "recall": 59.0,
-            "f1": 56.2,
-            "f2": 57.8,
-            "tp": 102,
-            "fp": 88,
-            "fn": 71,
-            "num_prs": 50
-          }
-        },
-        "cubic-v2": {
-          "strict": {
-            "precision": 52.9,
-            "recall": 59.7,
-            "f1": 56.1,
-            "f2": 58.2,
-            "tp": 83,
-            "fp": 74,
-            "fn": 56,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 56.5,
-            "recall": 60.8,
-            "f1": 58.5,
-            "f2": 59.9,
-            "tp": 96,
-            "fp": 74,
-            "fn": 62,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 57.2,
-            "recall": 57.2,
-            "f1": 57.2,
-            "f2": 57.2,
-            "tp": 99,
-            "fp": 74,
-            "fn": 74,
-            "num_prs": 50
-          }
-        },
-        "copilot-v2": {
-          "strict": {
-            "precision": 31.4,
-            "recall": 61.9,
-            "f1": 41.6,
-            "f2": 51.8,
-            "tp": 86,
-            "fp": 188,
-            "fn": 53,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 34.3,
-            "recall": 62.0,
-            "f1": 44.1,
-            "f2": 53.4,
-            "tp": 98,
-            "fp": 188,
-            "fn": 60,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 36.1,
-            "recall": 61.3,
-            "f1": 45.4,
-            "f2": 53.8,
-            "tp": 106,
-            "fp": 188,
-            "fn": 67,
-            "num_prs": 50
-          }
-        },
         "greptile-v4-1": {
           "strict": {
             "precision": 44.9,
@@ -36454,102 +36358,6 @@
             "num_prs": 50
           }
         },
-        "sourcery": {
-          "strict": {
-            "precision": 34.2,
-            "recall": 48.9,
-            "f1": 40.2,
-            "f2": 45.0,
-            "tp": 68,
-            "fp": 131,
-            "fn": 71,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 37.0,
-            "recall": 48.7,
-            "f1": 42.1,
-            "f2": 45.8,
-            "tp": 77,
-            "fp": 131,
-            "fn": 81,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 38.8,
-            "recall": 48.0,
-            "f1": 42.9,
-            "f2": 45.8,
-            "tp": 83,
-            "fp": 131,
-            "fn": 90,
-            "num_prs": 50
-          }
-        },
-        "qodo-v2": {
-          "strict": {
-            "precision": 45.1,
-            "recall": 59.0,
-            "f1": 51.1,
-            "f2": 55.6,
-            "tp": 82,
-            "fp": 100,
-            "fn": 57,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 47.1,
-            "recall": 56.3,
-            "f1": 51.3,
-            "f2": 54.2,
-            "tp": 89,
-            "fp": 100,
-            "fn": 69,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 47.9,
-            "recall": 53.2,
-            "f1": 50.4,
-            "f2": 52.0,
-            "tp": 92,
-            "fp": 100,
-            "fn": 81,
-            "num_prs": 50
-          }
-        },
-        "gitlab": {
-          "strict": {
-            "precision": 44.7,
-            "recall": 48.9,
-            "f1": 46.7,
-            "f2": 48.0,
-            "tp": 68,
-            "fp": 84,
-            "fn": 71,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 47.5,
-            "recall": 48.1,
-            "f1": 47.8,
-            "f2": 48.0,
-            "tp": 76,
-            "fp": 84,
-            "fn": 82,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 49.7,
-            "recall": 48.0,
-            "f1": 48.8,
-            "f2": 48.3,
-            "tp": 83,
-            "fp": 84,
-            "fn": 90,
-            "num_prs": 50
-          }
-        },
         "codeant-v2": {
           "strict": {
             "precision": 31.1,
@@ -36582,35 +36390,35 @@
             "num_prs": 50
           }
         },
-        "bugbot": {
+        "baz": {
           "strict": {
-            "precision": 52.0,
-            "recall": 46.0,
-            "f1": 48.9,
-            "f2": 47.1,
-            "tp": 64,
-            "fp": 59,
-            "fn": 75,
+            "precision": 54.6,
+            "recall": 38.1,
+            "f1": 44.9,
+            "f2": 40.6,
+            "tp": 53,
+            "fp": 44,
+            "fn": 86,
             "num_prs": 50
           },
           "core": {
-            "precision": 54.3,
-            "recall": 44.3,
-            "f1": 48.8,
-            "f2": 46.0,
-            "tp": 70,
-            "fp": 59,
-            "fn": 88,
+            "precision": 55.6,
+            "recall": 34.8,
+            "f1": 42.8,
+            "f2": 37.6,
+            "tp": 55,
+            "fp": 44,
+            "fn": 103,
             "num_prs": 50
           },
           "all": {
-            "precision": 55.3,
-            "recall": 42.2,
-            "f1": 47.9,
-            "f2": 44.3,
-            "tp": 73,
-            "fp": 59,
-            "fn": 100,
+            "precision": 56.9,
+            "recall": 33.5,
+            "f1": 42.2,
+            "f2": 36.5,
+            "tp": 58,
+            "fp": 44,
+            "fn": 115,
             "num_prs": 50
           }
         },
@@ -36646,6 +36454,102 @@
             "num_prs": 50
           }
         },
+        "kg": {
+          "strict": {
+            "precision": 48.8,
+            "recall": 15.1,
+            "f1": 23.1,
+            "f2": 17.5,
+            "tp": 21,
+            "fp": 22,
+            "fn": 118,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 50.0,
+            "recall": 13.9,
+            "f1": 21.8,
+            "f2": 16.3,
+            "tp": 22,
+            "fp": 22,
+            "fn": 136,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 51.1,
+            "recall": 13.3,
+            "f1": 21.1,
+            "f2": 15.6,
+            "tp": 23,
+            "fp": 22,
+            "fn": 150,
+            "num_prs": 50
+          }
+        },
+        "graphite": {
+          "strict": {
+            "precision": 100.0,
+            "recall": 7.9,
+            "f1": 14.7,
+            "f2": 9.7,
+            "tp": 11,
+            "fp": 0,
+            "fn": 128,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 100.0,
+            "recall": 7.6,
+            "f1": 14.1,
+            "f2": 9.3,
+            "tp": 12,
+            "fp": 0,
+            "fn": 146,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 100.0,
+            "recall": 6.9,
+            "f1": 13.0,
+            "f2": 8.5,
+            "tp": 12,
+            "fp": 0,
+            "fn": 161,
+            "num_prs": 50
+          }
+        },
+        "bugbot": {
+          "strict": {
+            "precision": 52.0,
+            "recall": 46.0,
+            "f1": 48.9,
+            "f2": 47.1,
+            "tp": 64,
+            "fp": 59,
+            "fn": 75,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 54.3,
+            "recall": 44.3,
+            "f1": 48.8,
+            "f2": 46.0,
+            "tp": 70,
+            "fp": 59,
+            "fn": 88,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 55.3,
+            "recall": 42.2,
+            "f1": 47.9,
+            "f2": 44.3,
+            "tp": 73,
+            "fp": 59,
+            "fn": 100,
+            "num_prs": 50
+          }
+        },
         "devin": {
           "strict": {
             "precision": 66.0,
@@ -36678,35 +36582,35 @@
             "num_prs": 50
           }
         },
-        "macroscope": {
+        "gitlab": {
           "strict": {
-            "precision": 50.4,
-            "recall": 45.3,
-            "f1": 47.7,
-            "f2": 46.3,
-            "tp": 63,
-            "fp": 62,
-            "fn": 76,
+            "precision": 44.7,
+            "recall": 48.9,
+            "f1": 46.7,
+            "f2": 48.0,
+            "tp": 68,
+            "fp": 84,
+            "fn": 71,
             "num_prs": 50
           },
           "core": {
-            "precision": 51.6,
-            "recall": 41.8,
-            "f1": 46.2,
-            "f2": 43.4,
-            "tp": 66,
-            "fp": 62,
-            "fn": 92,
+            "precision": 47.5,
+            "recall": 48.1,
+            "f1": 47.8,
+            "f2": 48.0,
+            "tp": 76,
+            "fp": 84,
+            "fn": 82,
             "num_prs": 50
           },
           "all": {
-            "precision": 53.0,
-            "recall": 40.5,
-            "f1": 45.9,
-            "f2": 42.5,
-            "tp": 70,
-            "fp": 62,
-            "fn": 103,
+            "precision": 49.7,
+            "recall": 48.0,
+            "f1": 48.8,
+            "f2": 48.3,
+            "tp": 83,
+            "fp": 84,
+            "fn": 90,
             "num_prs": 50
           }
         },
@@ -36742,227 +36646,2030 @@
             "num_prs": 50
           }
         },
-        "gemini-v2": {
+        "copilot-v2": {
           "strict": {
-            "precision": 36.7,
-            "recall": 51.8,
-            "f1": 43.0,
-            "f2": 47.9,
-            "tp": 72,
-            "fp": 124,
+            "precision": 31.4,
+            "recall": 61.9,
+            "f1": 41.6,
+            "f2": 51.8,
+            "tp": 86,
+            "fp": 188,
+            "fn": 53,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 34.3,
+            "recall": 62.0,
+            "f1": 44.1,
+            "f2": 53.4,
+            "tp": 98,
+            "fp": 188,
+            "fn": 60,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 36.1,
+            "recall": 61.3,
+            "f1": 45.4,
+            "f2": 53.8,
+            "tp": 106,
+            "fp": 188,
             "fn": 67,
             "num_prs": 50
+          }
+        },
+        "qodo-v2": {
+          "strict": {
+            "precision": 45.1,
+            "recall": 59.0,
+            "f1": 51.1,
+            "f2": 55.6,
+            "tp": 82,
+            "fp": 100,
+            "fn": 57,
+            "num_prs": 50
           },
           "core": {
-            "precision": 39.2,
-            "recall": 50.6,
-            "f1": 44.2,
-            "f2": 47.8,
-            "tp": 80,
-            "fp": 124,
-            "fn": 78,
+            "precision": 47.1,
+            "recall": 56.3,
+            "f1": 51.3,
+            "f2": 54.2,
+            "tp": 89,
+            "fp": 100,
+            "fn": 69,
             "num_prs": 50
           },
           "all": {
-            "precision": 41.2,
-            "recall": 50.3,
-            "f1": 45.3,
-            "f2": 48.2,
-            "tp": 87,
-            "fp": 124,
-            "fn": 86,
+            "precision": 47.9,
+            "recall": 53.2,
+            "f1": 50.4,
+            "f2": 52.0,
+            "tp": 92,
+            "fp": 100,
+            "fn": 81,
             "num_prs": 50
           }
         },
-        "coderabbit": {
+        "macroscope": {
           "strict": {
-            "precision": 31.8,
-            "recall": 60.4,
-            "f1": 41.7,
-            "f2": 51.2,
-            "tp": 84,
-            "fp": 180,
-            "fn": 55,
+            "precision": 50.4,
+            "recall": 45.3,
+            "f1": 47.7,
+            "f2": 46.3,
+            "tp": 63,
+            "fp": 62,
+            "fn": 76,
             "num_prs": 50
           },
           "core": {
-            "precision": 33.8,
-            "recall": 58.2,
-            "f1": 42.8,
-            "f2": 50.9,
+            "precision": 51.6,
+            "recall": 41.8,
+            "f1": 46.2,
+            "f2": 43.4,
+            "tp": 66,
+            "fp": 62,
+            "fn": 92,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 53.0,
+            "recall": 40.5,
+            "f1": 45.9,
+            "f2": 42.5,
+            "tp": 70,
+            "fp": 62,
+            "fn": 103,
+            "num_prs": 50
+          }
+        },
+        "qodo-extended-v2": {
+          "strict": {
+            "precision": 53.6,
+            "recall": 58.3,
+            "f1": 55.9,
+            "f2": 57.3,
+            "tp": 81,
+            "fp": 70,
+            "fn": 58,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 56.2,
+            "recall": 57.0,
+            "f1": 56.6,
+            "f2": 56.8,
+            "tp": 90,
+            "fp": 70,
+            "fn": 68,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 56.8,
+            "recall": 53.2,
+            "f1": 54.9,
+            "f2": 53.9,
             "tp": 92,
-            "fp": 180,
-            "fn": 66,
+            "fp": 70,
+            "fn": 81,
+            "num_prs": 50
+          }
+        },
+        "cubic-v2": {
+          "strict": {
+            "precision": 52.9,
+            "recall": 59.7,
+            "f1": 56.1,
+            "f2": 58.2,
+            "tp": 83,
+            "fp": 74,
+            "fn": 56,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 56.5,
+            "recall": 60.8,
+            "f1": 58.5,
+            "f2": 59.9,
+            "tp": 96,
+            "fp": 74,
+            "fn": 62,
             "num_prs": 50
           },
           "all": {
-            "precision": 35.3,
-            "recall": 56.6,
-            "f1": 43.5,
-            "f2": 50.5,
-            "tp": 98,
-            "fp": 180,
-            "fn": 75,
+            "precision": 57.2,
+            "recall": 57.2,
+            "f1": 57.2,
+            "f2": 57.2,
+            "tp": 99,
+            "fp": 74,
+            "fn": 74,
             "num_prs": 50
           }
         },
-        "baz": {
+        "kodus-v2": {
           "strict": {
-            "precision": 54.6,
+            "precision": 55.2,
             "recall": 38.1,
-            "f1": 44.9,
+            "f1": 45.1,
             "f2": 40.6,
             "tp": 53,
-            "fp": 44,
+            "fp": 43,
             "fn": 86,
             "num_prs": 50
           },
           "core": {
-            "precision": 55.6,
-            "recall": 34.8,
-            "f1": 42.8,
-            "f2": 37.6,
-            "tp": 55,
-            "fp": 44,
-            "fn": 103,
+            "precision": 57.4,
+            "recall": 36.7,
+            "f1": 44.8,
+            "f2": 39.6,
+            "tp": 58,
+            "fp": 43,
+            "fn": 100,
             "num_prs": 50
           },
           "all": {
-            "precision": 56.9,
-            "recall": 33.5,
-            "f1": 42.2,
-            "f2": 36.5,
-            "tp": 58,
-            "fp": 44,
-            "fn": 115,
+            "precision": 57.8,
+            "recall": 34.1,
+            "f1": 42.9,
+            "f2": 37.2,
+            "tp": 59,
+            "fp": 43,
+            "fn": 114,
             "num_prs": 50
           }
         },
-        "kg": {
-          "strict": {
-            "precision": 48.8,
-            "recall": 15.1,
-            "f1": 23.1,
-            "f2": 17.5,
-            "tp": 21,
-            "fp": 22,
-            "fn": 118,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 50.0,
-            "recall": 13.9,
-            "f1": 21.8,
-            "f2": 16.3,
-            "tp": 22,
-            "fp": 22,
-            "fn": 136,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 51.1,
-            "recall": 13.3,
-            "f1": 21.1,
-            "f2": 15.6,
-            "tp": 23,
-            "fp": 22,
-            "fn": 150,
-            "num_prs": 50
+        "sourcery": {
+          "strict": {
+            "precision": 34.2,
+            "recall": 48.9,
+            "f1": 40.2,
+            "f2": 45.0,
+            "tp": 68,
+            "fp": 131,
+            "fn": 71,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 37.0,
+            "recall": 48.7,
+            "f1": 42.1,
+            "f2": 45.8,
+            "tp": 77,
+            "fp": 131,
+            "fn": 81,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 38.8,
+            "recall": 48.0,
+            "f1": 42.9,
+            "f2": 45.8,
+            "tp": 83,
+            "fp": 131,
+            "fn": 90,
+            "num_prs": 50
+          }
+        },
+        "gemini-v2": {
+          "strict": {
+            "precision": 36.7,
+            "recall": 51.8,
+            "f1": 43.0,
+            "f2": 47.9,
+            "tp": 72,
+            "fp": 124,
+            "fn": 67,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 39.2,
+            "recall": 50.6,
+            "f1": 44.2,
+            "f2": 47.8,
+            "tp": 80,
+            "fp": 124,
+            "fn": 78,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 41.2,
+            "recall": 50.3,
+            "f1": 45.3,
+            "f2": 48.2,
+            "tp": 87,
+            "fp": 124,
+            "fn": 86,
+            "num_prs": 50
+          }
+        },
+        "augment": {
+          "strict": {
+            "precision": 49.4,
+            "recall": 61.9,
+            "f1": 55.0,
+            "f2": 58.9,
+            "tp": 86,
+            "fp": 88,
+            "fn": 53,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 52.4,
+            "recall": 61.4,
+            "f1": 56.6,
+            "f2": 59.4,
+            "tp": 97,
+            "fp": 88,
+            "fn": 61,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 53.7,
+            "recall": 59.0,
+            "f1": 56.2,
+            "f2": 57.8,
+            "tp": 102,
+            "fp": 88,
+            "fn": 71,
+            "num_prs": 50
+          }
+        },
+        "coderabbit": {
+          "strict": {
+            "precision": 31.8,
+            "recall": 60.4,
+            "f1": 41.7,
+            "f2": 51.2,
+            "tp": 84,
+            "fp": 180,
+            "fn": 55,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 33.8,
+            "recall": 58.2,
+            "f1": 42.8,
+            "f2": 50.9,
+            "tp": 92,
+            "fp": 180,
+            "fn": 66,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 35.3,
+            "recall": 56.6,
+            "f1": 43.5,
+            "f2": 50.5,
+            "tp": 98,
+            "fp": 180,
+            "fn": 75,
+            "num_prs": 50
+          }
+        }
+      }
+    },
+    "gpt-5.5": {
+      "prs": [
+        {
+          "url": "https://github.com/keycloak/keycloak/pull/37429",
+          "language": "Java",
+          "pr_size": "medium",
+          "domain": "UI",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "moderate",
+          "risk": "medium",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 1
+              },
+              "core": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 2
+              },
+              "all": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 3
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/keycloak/keycloak/pull/37634",
+          "language": "Java",
+          "pr_size": "medium",
+          "domain": "authentication",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 1
+              },
+              "core": {
+                "tp": 2,
+                "fp": 3,
+                "fn": 1
+              },
+              "all": {
+                "tp": 2,
+                "fp": 3,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/keycloak/keycloak/pull/38446",
+          "language": "Java",
+          "pr_size": "medium",
+          "domain": "authentication",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "moderate",
+          "risk": "medium",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 2
+              },
+              "core": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 2
+              },
+              "all": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 2
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/keycloak/keycloak/pull/36882",
+          "language": "Java",
+          "pr_size": "small",
+          "domain": "configuration",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "moderate",
+          "risk": "medium",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 1
+              },
+              "core": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 1
+              },
+              "all": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/keycloak/keycloak/pull/36880",
+          "language": "Java",
+          "pr_size": "medium",
+          "domain": "authentication",
+          "change_type": "feature",
+          "complexity": "complex",
+          "difficulty": "very_subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 6,
+                "fn": 3
+              },
+              "core": {
+                "tp": 0,
+                "fp": 6,
+                "fn": 3
+              },
+              "all": {
+                "tp": 0,
+                "fp": 6,
+                "fn": 3
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/keycloak/keycloak/pull/37038",
+          "language": "Java",
+          "pr_size": "small",
+          "domain": "authentication",
+          "change_type": "feature",
+          "complexity": "complex",
+          "difficulty": "very_subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 0
+              },
+              "all": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/keycloak/keycloak/pull/33832",
+          "language": "Java",
+          "pr_size": "medium",
+          "domain": "authentication",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/keycloak/keycloak/pull/40940",
+          "language": "Java",
+          "pr_size": "small",
+          "domain": "concurrency",
+          "change_type": "bug_fix",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1",
+          "language": "Java",
+          "pr_size": "medium",
+          "domain": "authentication",
+          "change_type": "bug_fix",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 0
+              },
+              "all": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/getsentry/sentry/pull/93824",
+          "language": "Python",
+          "pr_size": "large",
+          "domain": "concurrency",
+          "change_type": "feature",
+          "complexity": "complex",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 5,
+                "fn": 2
+              },
+              "core": {
+                "tp": 0,
+                "fp": 5,
+                "fn": 3
+              },
+              "all": {
+                "tp": 0,
+                "fp": 5,
+                "fn": 3
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5",
+          "language": "Python",
+          "pr_size": "large",
+          "domain": "API",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "all": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1",
+          "language": "Python",
+          "pr_size": "medium",
+          "domain": "API",
+          "change_type": "performance",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 2
+              },
+              "core": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 2
+              },
+              "all": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 2
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/grafana/grafana/pull/97529",
+          "language": "Go",
+          "pr_size": "small",
+          "domain": "concurrency",
+          "change_type": "performance",
+          "complexity": "complex",
+          "difficulty": "very_subtle",
+          "risk": "critical",
+          "context": "cross_file",
+          "concern": "reliability",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 2
+              },
+              "core": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 2
+              },
+              "all": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 2
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/getsentry/sentry/pull/80168",
+          "language": "Python",
+          "pr_size": "small",
+          "domain": "data_processing",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "moderate",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/getsentry/sentry/pull/80528",
+          "language": "Python",
+          "pr_size": "small",
+          "domain": "scheduling",
+          "change_type": "refactoring",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 1
+              },
+              "all": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/getsentry/sentry/pull/77754",
+          "language": "Python",
+          "pr_size": "medium",
+          "domain": "serialization",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "moderate",
+          "risk": "medium",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 1
+              },
+              "core": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 1
+              },
+              "all": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/getsentry/sentry/pull/95633",
+          "language": "Python",
+          "pr_size": "medium",
+          "domain": "concurrency",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "moderate",
+          "risk": "medium",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 4,
+                "fn": 0
+              },
+              "core": {
+                "tp": 0,
+                "fp": 4,
+                "fn": 1
+              },
+              "all": {
+                "tp": 0,
+                "fp": 4,
+                "fn": 2
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2",
+          "language": "Python",
+          "pr_size": "medium",
+          "domain": "data_processing",
+          "change_type": "performance",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 1
+              },
+              "core": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 1
+              },
+              "all": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3",
+          "language": "Python",
+          "pr_size": "medium",
+          "domain": "caching",
+          "change_type": "feature",
+          "complexity": "complex",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 2
+              },
+              "core": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 2
+              },
+              "all": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 2
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/grafana/grafana/pull/103633",
+          "language": "Go",
+          "pr_size": "small",
+          "domain": "caching",
+          "change_type": "performance",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "security",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 1
+              },
+              "core": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 2
+              },
+              "all": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 2
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/getsentry/sentry/pull/67876",
+          "language": "Python",
+          "pr_size": "small",
+          "domain": "authentication",
+          "change_type": "security_patch",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "security",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 6,
+                "fn": 2
+              },
+              "core": {
+                "tp": 1,
+                "fp": 6,
+                "fn": 2
+              },
+              "all": {
+                "tp": 1,
+                "fp": 6,
+                "fn": 2
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/keycloak/keycloak/pull/32918",
+          "language": "Java",
+          "pr_size": "small",
+          "domain": "caching",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 0
+              },
+              "all": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/grafana/grafana/pull/94942",
+          "language": "Go",
+          "pr_size": "small",
+          "domain": "data_processing",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "moderate",
+          "risk": "high",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 0
+              },
+              "all": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/grafana/grafana/pull/90939",
+          "language": "Go",
+          "pr_size": "small",
+          "domain": "caching",
+          "change_type": "performance",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 1
+              },
+              "core": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 1
+              },
+              "all": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/grafana/grafana/pull/80329",
+          "language": "Go",
+          "pr_size": "small",
+          "domain": "logging",
+          "change_type": "performance",
+          "complexity": "simple",
+          "difficulty": "obvious",
+          "risk": "low",
+          "context": "local",
+          "concern": "maintainability",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 0
+              },
+              "all": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/grafana/grafana/pull/90045",
+          "language": "Go",
+          "pr_size": "medium",
+          "domain": "logging",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "moderate",
+          "risk": "medium",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 2,
+                "fp": 7,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 7,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 7,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/grafana/grafana/pull/106778",
+          "language": "Go",
+          "pr_size": "medium",
+          "domain": "UI",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 4,
+                "fn": 1
+              },
+              "core": {
+                "tp": 1,
+                "fp": 4,
+                "fn": 1
+              },
+              "all": {
+                "tp": 1,
+                "fp": 4,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/grafana/grafana/pull/107534",
+          "language": "Go",
+          "pr_size": "small",
+          "domain": "testing",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "low",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 1
+              },
+              "all": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/grafana/grafana/pull/79265",
+          "language": "Go",
+          "pr_size": "large",
+          "domain": "authentication",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 3
+              },
+              "core": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 3
+              },
+              "all": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 3
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9",
+          "language": "Ruby",
+          "pr_size": "small",
+          "domain": "configuration",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "file",
+          "concern": "reliability",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 0
+              },
+              "core": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 1
+              },
+              "all": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/grafana/grafana/pull/76186",
+          "language": "Go",
+          "pr_size": "small",
+          "domain": "logging",
+          "change_type": "refactoring",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "reliability",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 2
+              },
+              "core": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 2
+              },
+              "all": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 2
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10",
+          "language": "Ruby",
+          "pr_size": "large",
+          "domain": "configuration",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 2,
+                "fp": 7,
+                "fn": 1
+              },
+              "core": {
+                "tp": 2,
+                "fp": 7,
+                "fn": 1
+              },
+              "all": {
+                "tp": 2,
+                "fp": 7,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7",
+          "language": "Ruby",
+          "pr_size": "medium",
+          "domain": "UI",
+          "change_type": "bug_fix",
+          "complexity": "simple",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "all": {
+                "tp": 3,
+                "fp": 2,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8",
+          "language": "Ruby",
+          "pr_size": "medium",
+          "domain": "API",
+          "change_type": "bug_fix",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 4,
+                "fn": 1
+              },
+              "core": {
+                "tp": 1,
+                "fp": 4,
+                "fn": 1
+              },
+              "all": {
+                "tp": 1,
+                "fp": 4,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3",
+          "language": "Ruby",
+          "pr_size": "medium",
+          "domain": "authentication",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "all": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5",
+          "language": "Ruby",
+          "pr_size": "small",
+          "domain": "UI",
+          "change_type": "performance",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 1
+              },
+              "core": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 1
+              },
+              "all": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6",
+          "language": "Ruby",
+          "pr_size": "small",
+          "domain": "serialization",
+          "change_type": "feature",
+          "complexity": "simple",
+          "difficulty": "moderate",
+          "risk": "medium",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 1
+              },
+              "core": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 1
+              },
+              "all": {
+                "tp": 0,
+                "fp": 3,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4",
+          "language": "Ruby",
+          "pr_size": "large",
+          "domain": "API",
+          "change_type": "feature",
+          "complexity": "complex",
+          "difficulty": "subtle",
+          "risk": "critical",
+          "context": "cross_file",
+          "concern": "security",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 3,
+                "fp": 6,
+                "fn": 2
+              },
+              "core": {
+                "tp": 3,
+                "fp": 6,
+                "fn": 2
+              },
+              "all": {
+                "tp": 3,
+                "fp": 6,
+                "fn": 2
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1",
+          "language": "Ruby",
+          "pr_size": "medium",
+          "domain": "file_io",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 2,
+                "fp": 1,
+                "fn": 1
+              },
+              "core": {
+                "tp": 2,
+                "fp": 1,
+                "fn": 1
+              },
+              "all": {
+                "tp": 2,
+                "fp": 1,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2",
+          "language": "Ruby",
+          "pr_size": "medium",
+          "domain": "UI",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "moderate",
+          "risk": "medium",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/calcom/cal.com/pull/22532",
+          "language": "TypeScript",
+          "pr_size": "medium",
+          "domain": "caching",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/calcom/cal.com/pull/8330",
+          "language": "TypeScript",
+          "pr_size": "small",
+          "domain": "scheduling",
+          "change_type": "bug_fix",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 2,
+                "fp": 6,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 6,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 6,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/calcom/cal.com/pull/14943",
+          "language": "TypeScript",
+          "pr_size": "small",
+          "domain": "scheduling",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/calcom/cal.com/pull/22345",
+          "language": "TypeScript",
+          "pr_size": "small",
+          "domain": "database",
+          "change_type": "migration",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "medium",
+          "context": "file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 0
+              },
+              "all": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 1
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/calcom/cal.com/pull/11059",
+          "language": "TypeScript",
+          "pr_size": "large",
+          "domain": "authentication",
+          "change_type": "feature",
+          "complexity": "complex",
+          "difficulty": "subtle",
+          "risk": "critical",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 4,
+                "fp": 9,
+                "fn": 0
+              },
+              "core": {
+                "tp": 4,
+                "fp": 9,
+                "fn": 0
+              },
+              "all": {
+                "tp": 4,
+                "fp": 9,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/calcom/cal.com/pull/7232",
+          "language": "TypeScript",
+          "pr_size": "medium",
+          "domain": "scheduling",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "reliability",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 2,
+                "fp": 3,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 3,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 3,
+                "fn": 0
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/calcom/cal.com/pull/14740",
+          "language": "TypeScript",
+          "pr_size": "large",
+          "domain": "scheduling",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "critical",
+          "context": "cross_file",
+          "concern": "security",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 2
+              },
+              "core": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 2
+              },
+              "all": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 2
+              }
+            }
+          }
+        },
+        {
+          "url": "https://github.com/calcom/cal.com/pull/10600",
+          "language": "TypeScript",
+          "pr_size": "medium",
+          "domain": "authentication",
+          "change_type": "feature",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "security",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 10,
+                "fn": 2
+              },
+              "core": {
+                "tp": 1,
+                "fp": 10,
+                "fn": 2
+              },
+              "all": {
+                "tp": 2,
+                "fp": 10,
+                "fn": 2
+              }
+            }
           }
         },
-        "qodo-extended-v2": {
-          "strict": {
-            "precision": 53.6,
-            "recall": 58.3,
-            "f1": 55.9,
-            "f2": 57.3,
-            "tp": 81,
-            "fp": 70,
-            "fn": 58,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 56.2,
-            "recall": 57.0,
-            "f1": 56.6,
-            "f2": 56.8,
-            "tp": 90,
-            "fp": 70,
-            "fn": 68,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 56.8,
-            "recall": 53.2,
-            "f1": 54.9,
-            "f2": 53.9,
-            "tp": 92,
-            "fp": 70,
-            "fn": 81,
-            "num_prs": 50
+        {
+          "url": "https://github.com/calcom/cal.com/pull/10967",
+          "language": "TypeScript",
+          "pr_size": "large",
+          "domain": "scheduling",
+          "change_type": "bug_fix",
+          "complexity": "complex",
+          "difficulty": "subtle",
+          "risk": "high",
+          "context": "cross_file",
+          "concern": "correctness",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 0,
+                "fp": 6,
+                "fn": 3
+              },
+              "core": {
+                "tp": 0,
+                "fp": 6,
+                "fn": 3
+              },
+              "all": {
+                "tp": 0,
+                "fp": 6,
+                "fn": 4
+              }
+            }
           }
         },
-        "kodus-v2": {
-          "strict": {
-            "precision": 55.2,
-            "recall": 38.1,
-            "f1": 45.1,
-            "f2": 40.6,
-            "tp": 53,
-            "fp": 43,
-            "fn": 86,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 57.4,
-            "recall": 36.7,
-            "f1": 44.8,
-            "f2": 39.6,
-            "tp": 58,
-            "fp": 43,
-            "fn": 100,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 57.8,
-            "recall": 34.1,
-            "f1": 42.9,
-            "f2": 37.2,
-            "tp": 59,
-            "fp": 43,
-            "fn": 114,
-            "num_prs": 50
+        {
+          "url": "https://github.com/calcom/cal.com/pull/8087",
+          "language": "TypeScript",
+          "pr_size": "medium",
+          "domain": "concurrency",
+          "change_type": "refactoring",
+          "complexity": "moderate",
+          "difficulty": "subtle",
+          "risk": "critical",
+          "context": "cross_file",
+          "concern": "reliability",
+          "summary": "",
+          "tool_metrics": {
+            "deputydev-v1": {
+              "strict": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 0
+              },
+              "all": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 1
+              }
+            }
           }
-        },
-        "graphite": {
+        }
+      ],
+      "tools": [
+        "deputydev-v1"
+      ],
+      "dimensions": {
+        "language": [
+          "Go",
+          "Java",
+          "Python",
+          "Ruby",
+          "TypeScript"
+        ],
+        "pr_size": [
+          "small",
+          "medium",
+          "large"
+        ],
+        "domain": [
+          "API",
+          "UI",
+          "authentication",
+          "caching",
+          "concurrency",
+          "configuration",
+          "data_processing",
+          "database",
+          "file_io",
+          "logging",
+          "scheduling",
+          "serialization",
+          "testing"
+        ],
+        "change_type": [
+          "bug_fix",
+          "feature",
+          "migration",
+          "performance",
+          "refactoring",
+          "security_patch"
+        ],
+        "complexity": [
+          "simple",
+          "moderate",
+          "complex"
+        ],
+        "difficulty": [
+          "obvious",
+          "moderate",
+          "subtle",
+          "very_subtle"
+        ],
+        "risk": [
+          "low",
+          "medium",
+          "high",
+          "critical"
+        ],
+        "context": [
+          "local",
+          "file",
+          "cross_file",
+          "system"
+        ],
+        "concern": [
+          "correctness",
+          "maintainability",
+          "reliability",
+          "security"
+        ]
+      },
+      "overall_metrics": {
+        "deputydev-v1": {
           "strict": {
-            "precision": 100.0,
-            "recall": 7.9,
-            "f1": 14.7,
-            "f2": 9.7,
-            "tp": 11,
-            "fp": 0,
-            "fn": 128,
+            "precision": 22.9,
+            "recall": 50.6,
+            "f1": 31.5,
+            "f2": 40.7,
+            "tp": 43,
+            "fp": 145,
+            "fn": 42,
             "num_prs": 50
           },
           "core": {
-            "precision": 100.0,
-            "recall": 7.6,
-            "f1": 14.1,
-            "f2": 9.3,
-            "tp": 12,
-            "fp": 0,
-            "fn": 146,
+            "precision": 25.3,
+            "recall": 50.0,
+            "f1": 33.6,
+            "f2": 41.8,
+            "tp": 49,
+            "fp": 145,
+            "fn": 49,
             "num_prs": 50
           },
           "all": {
-            "precision": 100.0,
-            "recall": 6.9,
-            "f1": 13.0,
-            "f2": 8.5,
-            "tp": 12,
-            "fp": 0,
-            "fn": 161,
+            "precision": 27.5,
+            "recall": 50.0,
+            "f1": 35.5,
+            "f2": 43.0,
+            "tp": 55,
+            "fp": 145,
+            "fn": 55,
             "num_prs": 50
           }
         }
@@ -54810,102 +56517,6 @@
         ]
       },
       "overall_metrics": {
-        "augment": {
-          "strict": {
-            "precision": 45.9,
-            "recall": 59.7,
-            "f1": 51.9,
-            "f2": 56.3,
-            "tp": 83,
-            "fp": 98,
-            "fn": 56,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 49.5,
-            "recall": 60.8,
-            "f1": 54.5,
-            "f2": 58.1,
-            "tp": 96,
-            "fp": 98,
-            "fn": 62,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 50.8,
-            "recall": 58.4,
-            "f1": 54.3,
-            "f2": 56.7,
-            "tp": 101,
-            "fp": 98,
-            "fn": 72,
-            "num_prs": 50
-          }
-        },
-        "cubic-v2": {
-          "strict": {
-            "precision": 53.2,
-            "recall": 59.7,
-            "f1": 56.3,
-            "f2": 58.3,
-            "tp": 83,
-            "fp": 73,
-            "fn": 56,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 56.8,
-            "recall": 60.8,
-            "f1": 58.7,
-            "f2": 59.9,
-            "tp": 96,
-            "fp": 73,
-            "fn": 62,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 57.8,
-            "recall": 57.8,
-            "f1": 57.8,
-            "f2": 57.8,
-            "tp": 100,
-            "fp": 73,
-            "fn": 73,
-            "num_prs": 50
-          }
-        },
-        "copilot-v2": {
-          "strict": {
-            "precision": 30.4,
-            "recall": 63.3,
-            "f1": 41.1,
-            "f2": 52.1,
-            "tp": 88,
-            "fp": 201,
-            "fn": 51,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 33.2,
-            "recall": 63.3,
-            "f1": 43.6,
-            "f2": 53.6,
-            "tp": 100,
-            "fp": 201,
-            "fn": 58,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 35.2,
-            "recall": 63.0,
-            "f1": 45.1,
-            "f2": 54.4,
-            "tp": 109,
-            "fp": 201,
-            "fn": 64,
-            "num_prs": 50
-          }
-        },
         "greptile-v4-1": {
           "strict": {
             "precision": 40.2,
@@ -54938,131 +56549,163 @@
             "num_prs": 50
           }
         },
-        "sourcery": {
+        "codeant-v2": {
           "strict": {
-            "precision": 33.3,
-            "recall": 50.4,
-            "f1": 40.1,
-            "f2": 45.7,
-            "tp": 70,
-            "fp": 140,
-            "fn": 69,
+            "precision": 29.3,
+            "recall": 38.1,
+            "f1": 33.1,
+            "f2": 36.0,
+            "tp": 53,
+            "fp": 128,
+            "fn": 86,
             "num_prs": 50
           },
           "core": {
-            "precision": 36.1,
-            "recall": 50.0,
-            "f1": 41.9,
-            "f2": 46.4,
-            "tp": 79,
-            "fp": 140,
-            "fn": 79,
+            "precision": 30.4,
+            "recall": 35.4,
+            "f1": 32.7,
+            "f2": 34.3,
+            "tp": 56,
+            "fp": 128,
+            "fn": 102,
             "num_prs": 50
           },
           "all": {
-            "precision": 38.1,
-            "recall": 49.7,
-            "f1": 43.1,
-            "f2": 46.8,
-            "tp": 86,
-            "fp": 140,
-            "fn": 87,
+            "precision": 31.2,
+            "recall": 33.5,
+            "f1": 32.3,
+            "f2": 33.0,
+            "tp": 58,
+            "fp": 128,
+            "fn": 115,
             "num_prs": 50
           }
         },
-        "qodo-v2": {
+        "baz": {
           "strict": {
-            "precision": 44.1,
-            "recall": 64.7,
-            "f1": 52.5,
-            "f2": 59.2,
-            "tp": 90,
-            "fp": 114,
-            "fn": 49,
+            "precision": 43.9,
+            "recall": 36.0,
+            "f1": 39.5,
+            "f2": 37.3,
+            "tp": 50,
+            "fp": 64,
+            "fn": 89,
             "num_prs": 50
           },
           "core": {
-            "precision": 46.5,
-            "recall": 62.7,
-            "f1": 53.4,
-            "f2": 58.6,
-            "tp": 99,
-            "fp": 114,
-            "fn": 59,
+            "precision": 44.8,
+            "recall": 32.9,
+            "f1": 38.0,
+            "f2": 34.8,
+            "tp": 52,
+            "fp": 64,
+            "fn": 106,
             "num_prs": 50
           },
           "all": {
-            "precision": 47.5,
-            "recall": 59.5,
-            "f1": 52.8,
-            "f2": 56.7,
-            "tp": 103,
-            "fp": 114,
-            "fn": 70,
+            "precision": 46.2,
+            "recall": 31.8,
+            "f1": 37.7,
+            "f2": 33.9,
+            "tp": 55,
+            "fp": 64,
+            "fn": 118,
+            "num_prs": 50
+          }
+        },
+        "claude-code": {
+          "strict": {
+            "precision": 38.2,
+            "recall": 43.2,
+            "f1": 40.5,
+            "f2": 42.1,
+            "tp": 60,
+            "fp": 97,
+            "fn": 79,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 40.1,
+            "recall": 41.1,
+            "f1": 40.6,
+            "f2": 40.9,
+            "tp": 65,
+            "fp": 97,
+            "fn": 93,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 42.3,
+            "recall": 41.0,
+            "f1": 41.6,
+            "f2": 41.3,
+            "tp": 71,
+            "fp": 97,
+            "fn": 102,
             "num_prs": 50
           }
         },
-        "gitlab": {
+        "kg": {
           "strict": {
-            "precision": 42.2,
-            "recall": 48.9,
-            "f1": 45.3,
-            "f2": 47.4,
-            "tp": 68,
-            "fp": 93,
-            "fn": 71,
+            "precision": 53.5,
+            "recall": 16.5,
+            "f1": 25.3,
+            "f2": 19.2,
+            "tp": 23,
+            "fp": 20,
+            "fn": 116,
             "num_prs": 50
           },
           "core": {
-            "precision": 45.0,
-            "recall": 48.1,
-            "f1": 46.5,
-            "f2": 47.4,
-            "tp": 76,
-            "fp": 93,
-            "fn": 82,
+            "precision": 54.5,
+            "recall": 15.2,
+            "f1": 23.8,
+            "f2": 17.8,
+            "tp": 24,
+            "fp": 20,
+            "fn": 134,
             "num_prs": 50
           },
           "all": {
-            "precision": 46.9,
-            "recall": 47.4,
-            "f1": 47.1,
-            "f2": 47.3,
-            "tp": 82,
-            "fp": 93,
-            "fn": 91,
+            "precision": 55.6,
+            "recall": 14.5,
+            "f1": 22.9,
+            "f2": 17.0,
+            "tp": 25,
+            "fp": 20,
+            "fn": 148,
             "num_prs": 50
           }
         },
-        "codeant-v2": {
+        "graphite": {
           "strict": {
-            "precision": 29.3,
-            "recall": 38.1,
-            "f1": 33.1,
-            "f2": 36.0,
-            "tp": 53,
-            "fp": 128,
-            "fn": 86,
+            "precision": 71.4,
+            "recall": 7.2,
+            "f1": 13.1,
+            "f2": 8.8,
+            "tp": 10,
+            "fp": 4,
+            "fn": 129,
             "num_prs": 50
           },
           "core": {
-            "precision": 30.4,
-            "recall": 35.4,
-            "f1": 32.7,
-            "f2": 34.3,
-            "tp": 56,
-            "fp": 128,
-            "fn": 102,
+            "precision": 73.3,
+            "recall": 7.0,
+            "f1": 12.7,
+            "f2": 8.5,
+            "tp": 11,
+            "fp": 4,
+            "fn": 147,
             "num_prs": 50
           },
           "all": {
-            "precision": 31.2,
-            "recall": 33.5,
-            "f1": 32.3,
-            "f2": 33.0,
-            "tp": 58,
-            "fp": 128,
-            "fn": 115,
+            "precision": 73.3,
+            "recall": 6.4,
+            "f1": 11.7,
+            "f2": 7.8,
+            "tp": 11,
+            "fp": 4,
+            "fn": 162,
             "num_prs": 50
           }
         },
@@ -55098,38 +56741,6 @@
             "num_prs": 50
           }
         },
-        "claude-code": {
-          "strict": {
-            "precision": 38.2,
-            "recall": 43.2,
-            "f1": 40.5,
-            "f2": 42.1,
-            "tp": 60,
-            "fp": 97,
-            "fn": 79,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 40.1,
-            "recall": 41.1,
-            "f1": 40.6,
-            "f2": 40.9,
-            "tp": 65,
-            "fp": 97,
-            "fn": 93,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 42.3,
-            "recall": 41.0,
-            "f1": 41.6,
-            "f2": 41.3,
-            "tp": 71,
-            "fp": 97,
-            "fn": 102,
-            "num_prs": 50
-          }
-        },
         "devin": {
           "strict": {
             "precision": 56.6,
@@ -55162,35 +56773,35 @@
             "num_prs": 50
           }
         },
-        "macroscope": {
+        "gitlab": {
           "strict": {
-            "precision": 47.7,
-            "recall": 44.6,
-            "f1": 46.1,
-            "f2": 45.2,
-            "tp": 62,
-            "fp": 68,
-            "fn": 77,
+            "precision": 42.2,
+            "recall": 48.9,
+            "f1": 45.3,
+            "f2": 47.4,
+            "tp": 68,
+            "fp": 93,
+            "fn": 71,
             "num_prs": 50
           },
           "core": {
-            "precision": 48.9,
-            "recall": 41.1,
-            "f1": 44.7,
-            "f2": 42.5,
-            "tp": 65,
-            "fp": 68,
-            "fn": 93,
+            "precision": 45.0,
+            "recall": 48.1,
+            "f1": 46.5,
+            "f2": 47.4,
+            "tp": 76,
+            "fp": 93,
+            "fn": 82,
             "num_prs": 50
           },
           "all": {
-            "precision": 50.0,
-            "recall": 39.3,
-            "f1": 44.0,
-            "f2": 41.1,
-            "tp": 68,
-            "fp": 68,
-            "fn": 105,
+            "precision": 46.9,
+            "recall": 47.4,
+            "f1": 47.1,
+            "f2": 47.3,
+            "tp": 82,
+            "fp": 93,
+            "fn": 91,
             "num_prs": 50
           }
         },
@@ -55226,131 +56837,99 @@
             "num_prs": 50
           }
         },
-        "gemini-v2": {
+        "copilot-v2": {
           "strict": {
-            "precision": 36.2,
-            "recall": 54.0,
-            "f1": 43.4,
-            "f2": 49.1,
-            "tp": 75,
-            "fp": 132,
-            "fn": 64,
+            "precision": 30.4,
+            "recall": 63.3,
+            "f1": 41.1,
+            "f2": 52.1,
+            "tp": 88,
+            "fp": 201,
+            "fn": 51,
             "num_prs": 50
           },
           "core": {
-            "precision": 38.0,
-            "recall": 51.3,
-            "f1": 43.7,
-            "f2": 47.9,
-            "tp": 81,
-            "fp": 132,
-            "fn": 77,
+            "precision": 33.2,
+            "recall": 63.3,
+            "f1": 43.6,
+            "f2": 53.6,
+            "tp": 100,
+            "fp": 201,
+            "fn": 58,
             "num_prs": 50
           },
           "all": {
-            "precision": 40.3,
-            "recall": 51.4,
-            "f1": 45.2,
-            "f2": 48.7,
-            "tp": 89,
-            "fp": 132,
-            "fn": 84,
-            "num_prs": 50
-          }
-        },
-        "coderabbit": {
-          "strict": {
-            "precision": 26.8,
-            "recall": 61.9,
-            "f1": 37.4,
-            "f2": 49.0,
-            "tp": 86,
-            "fp": 235,
-            "fn": 53,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 28.6,
-            "recall": 59.5,
-            "f1": 38.6,
-            "f2": 48.9,
-            "tp": 94,
-            "fp": 235,
+            "precision": 35.2,
+            "recall": 63.0,
+            "f1": 45.1,
+            "f2": 54.4,
+            "tp": 109,
+            "fp": 201,
             "fn": 64,
             "num_prs": 50
-          },
-          "all": {
-            "precision": 30.7,
-            "recall": 60.1,
-            "f1": 40.6,
-            "f2": 50.4,
-            "tp": 104,
-            "fp": 235,
-            "fn": 69,
-            "num_prs": 50
           }
         },
-        "baz": {
+        "qodo-v2": {
           "strict": {
-            "precision": 43.9,
-            "recall": 36.0,
-            "f1": 39.5,
-            "f2": 37.3,
-            "tp": 50,
-            "fp": 64,
-            "fn": 89,
+            "precision": 44.1,
+            "recall": 64.7,
+            "f1": 52.5,
+            "f2": 59.2,
+            "tp": 90,
+            "fp": 114,
+            "fn": 49,
             "num_prs": 50
           },
           "core": {
-            "precision": 44.8,
-            "recall": 32.9,
-            "f1": 38.0,
-            "f2": 34.8,
-            "tp": 52,
-            "fp": 64,
-            "fn": 106,
+            "precision": 46.5,
+            "recall": 62.7,
+            "f1": 53.4,
+            "f2": 58.6,
+            "tp": 99,
+            "fp": 114,
+            "fn": 59,
             "num_prs": 50
           },
           "all": {
-            "precision": 46.2,
-            "recall": 31.8,
-            "f1": 37.7,
-            "f2": 33.9,
-            "tp": 55,
-            "fp": 64,
-            "fn": 118,
+            "precision": 47.5,
+            "recall": 59.5,
+            "f1": 52.8,
+            "f2": 56.7,
+            "tp": 103,
+            "fp": 114,
+            "fn": 70,
             "num_prs": 50
           }
         },
-        "kg": {
+        "macroscope": {
           "strict": {
-            "precision": 53.5,
-            "recall": 16.5,
-            "f1": 25.3,
-            "f2": 19.2,
-            "tp": 23,
-            "fp": 20,
-            "fn": 116,
+            "precision": 47.7,
+            "recall": 44.6,
+            "f1": 46.1,
+            "f2": 45.2,
+            "tp": 62,
+            "fp": 68,
+            "fn": 77,
             "num_prs": 50
           },
           "core": {
-            "precision": 54.5,
-            "recall": 15.2,
-            "f1": 23.8,
-            "f2": 17.8,
-            "tp": 24,
-            "fp": 20,
-            "fn": 134,
+            "precision": 48.9,
+            "recall": 41.1,
+            "f1": 44.7,
+            "f2": 42.5,
+            "tp": 65,
+            "fp": 68,
+            "fn": 93,
             "num_prs": 50
           },
           "all": {
-            "precision": 55.6,
-            "recall": 14.5,
-            "f1": 22.9,
-            "f2": 17.0,
-            "tp": 25,
-            "fp": 20,
-            "fn": 148,
+            "precision": 50.0,
+            "recall": 39.3,
+            "f1": 44.0,
+            "f2": 41.1,
+            "tp": 68,
+            "fp": 68,
+            "fn": 105,
             "num_prs": 50
           }
         },
@@ -55386,6 +56965,38 @@
             "num_prs": 50
           }
         },
+        "cubic-v2": {
+          "strict": {
+            "precision": 53.2,
+            "recall": 59.7,
+            "f1": 56.3,
+            "f2": 58.3,
+            "tp": 83,
+            "fp": 73,
+            "fn": 56,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 56.8,
+            "recall": 60.8,
+            "f1": 58.7,
+            "f2": 59.9,
+            "tp": 96,
+            "fp": 73,
+            "fn": 62,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 57.8,
+            "recall": 57.8,
+            "f1": 57.8,
+            "f2": 57.8,
+            "tp": 100,
+            "fp": 73,
+            "fn": 73,
+            "num_prs": 50
+          }
+        },
         "kodus-v2": {
           "strict": {
             "precision": 50.5,
@@ -55418,35 +57029,131 @@
             "num_prs": 50
           }
         },
-        "graphite": {
+        "gemini-v2": {
           "strict": {
-            "precision": 71.4,
-            "recall": 7.2,
-            "f1": 13.1,
-            "f2": 8.8,
-            "tp": 10,
-            "fp": 4,
-            "fn": 129,
+            "precision": 36.2,
+            "recall": 54.0,
+            "f1": 43.4,
+            "f2": 49.1,
+            "tp": 75,
+            "fp": 132,
+            "fn": 64,
             "num_prs": 50
           },
           "core": {
-            "precision": 73.3,
-            "recall": 7.0,
-            "f1": 12.7,
-            "f2": 8.5,
-            "tp": 11,
-            "fp": 4,
-            "fn": 147,
+            "precision": 38.0,
+            "recall": 51.3,
+            "f1": 43.7,
+            "f2": 47.9,
+            "tp": 81,
+            "fp": 132,
+            "fn": 77,
             "num_prs": 50
           },
           "all": {
-            "precision": 73.3,
-            "recall": 6.4,
-            "f1": 11.7,
-            "f2": 7.8,
-            "tp": 11,
-            "fp": 4,
-            "fn": 162,
+            "precision": 40.3,
+            "recall": 51.4,
+            "f1": 45.2,
+            "f2": 48.7,
+            "tp": 89,
+            "fp": 132,
+            "fn": 84,
+            "num_prs": 50
+          }
+        },
+        "sourcery": {
+          "strict": {
+            "precision": 33.3,
+            "recall": 50.4,
+            "f1": 40.1,
+            "f2": 45.7,
+            "tp": 70,
+            "fp": 140,
+            "fn": 69,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 36.1,
+            "recall": 50.0,
+            "f1": 41.9,
+            "f2": 46.4,
+            "tp": 79,
+            "fp": 140,
+            "fn": 79,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 38.1,
+            "recall": 49.7,
+            "f1": 43.1,
+            "f2": 46.8,
+            "tp": 86,
+            "fp": 140,
+            "fn": 87,
+            "num_prs": 50
+          }
+        },
+        "augment": {
+          "strict": {
+            "precision": 45.9,
+            "recall": 59.7,
+            "f1": 51.9,
+            "f2": 56.3,
+            "tp": 83,
+            "fp": 98,
+            "fn": 56,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 49.5,
+            "recall": 60.8,
+            "f1": 54.5,
+            "f2": 58.1,
+            "tp": 96,
+            "fp": 98,
+            "fn": 62,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 50.8,
+            "recall": 58.4,
+            "f1": 54.3,
+            "f2": 56.7,
+            "tp": 101,
+            "fp": 98,
+            "fn": 72,
+            "num_prs": 50
+          }
+        },
+        "coderabbit": {
+          "strict": {
+            "precision": 26.8,
+            "recall": 61.9,
+            "f1": 37.4,
+            "f2": 49.0,
+            "tp": 86,
+            "fp": 235,
+            "fn": 53,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 28.6,
+            "recall": 59.5,
+            "f1": 38.6,
+            "f2": 48.9,
+            "tp": 94,
+            "fp": 235,
+            "fn": 64,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 30.7,
+            "recall": 60.1,
+            "f1": 40.6,
+            "f2": 50.4,
+            "tp": 104,
+            "fp": 235,
+            "fn": 69,
             "num_prs": 50
           }
         }
@@ -55616,6 +57323,22 @@
       "best_tool": "baz",
       "best_score": 74.1
     },
+    {
+      "id": "tool_deputydev-v1_risk_high_context_file",
+      "label": "High Risk + File Context",
+      "filters": {
+        "risk": [
+          "high"
+        ],
+        "context": [
+          "file"
+        ]
+      },
+      "description": "Significant impact, potential data loss. Requires full file understanding.",
+      "best_model": "gpt-5.5",
+      "best_tool": "deputydev-v1",
+      "best_score": 73.7
+    },
     {
       "id": "python_medium",
       "label": "Best for Medium Python PRs",
@@ -56148,7 +57871,8 @@
     "gitlab": "GitLab Duo Code Review",
     "qodo-v2": "Qodo v2",
     "qodo-extended-v2": "Qodo Extended",
-    "macroscope": "Macroscope"
+    "macroscope": "Macroscope",
+    "deputydev-v1": "DeputyDev v1"
   },
   "tool_colors": {
     "graphite": "#6366f1",
@@ -56177,7 +57901,8 @@
     "qodo-v2": "#7c3aed",
     "qodo-extended-v2": "#6d28d9",
     "macroscope": "#0891b2",
-    "cubic-v2": "#c026d3"
+    "cubic-v2": "#c026d3",
+    "deputydev-v1": "#2563eb"
   },
   "default_model": "anthropic_claude-opus-4-5-20251101",
   "min_prs_threshold": 5,
diff --git a/offline/analysis/benchmark_dashboard.py b/offline/analysis/benchmark_dashboard.py
index 559ba8c..03df603 100644
--- a/offline/analysis/benchmark_dashboard.py
+++ b/offline/analysis/benchmark_dashboard.py
@@ -162,6 +162,7 @@ def _compute_profile_metrics(
     "qodo-extended-v2": "Qodo Extended",
     "macroscope": "Macroscope",
     "cubic-v2": "Cubic v2",
+    "deputydev-v1": "DeputyDev v1",
 }
 
 TOOL_COLORS = {
@@ -192,6 +193,7 @@ def _compute_profile_metrics(
     "qodo-extended-v2": "#6d28d9",
     "macroscope": "#0891b2",
     "cubic-v2": "#c026d3",
+    "deputydev-v1": "#2563eb",
 }
 
 
diff --git a/offline/code_review_benchmark/step1_download_prs.py b/offline/code_review_benchmark/step1_download_prs.py
index cedf882..ae05fbf 100644
--- a/offline/code_review_benchmark/step1_download_prs.py
+++ b/offline/code_review_benchmark/step1_download_prs.py
@@ -108,7 +108,7 @@ def _is_bot(user: dict | None) -> bool:
 
 # Tools that post their review comments as a human GitHub account rather than a bot.
 # For these, bot filtering is skipped and all human comments are collected.
-_NON_BOT_TOOLS: frozenset[str] = frozenset({"claude"})
+_NON_BOT_TOOLS: frozenset[str] = frozenset({"claude", "deputydev-v1"})
 
 
 def fetch_review_comments(org: str, repo: str, pr: int, tool: str = "") -> list[dict]:
diff --git a/offline/results/benchmark_data.json b/offline/results/benchmark_data.json
index f6a1028..e4f5174 100644
--- a/offline/results/benchmark_data.json
+++ b/offline/results/benchmark_data.json
@@ -1789,6 +1789,37 @@
             "created_at": "2026-06-28T23:12:51Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "keycloak__keycloak__deputydev-v5__PR37429__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR37429__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java",
+            "line": 58,
+            "body": "**PERFORMANCE_OPTIMIZATION**: These policy factories and the reusable `HTML_TAGS` pattern do not depend on instance state, but they are initialized for every `VerifyMessageProperties` instance. Declare the policies and pattern as `private static final` constants so their builders and regex compilation run only once. \n Creating a verifier per properties file repeatedly rebuilds the same immutable policies and recompiles the same regular expression.\n```\nprivate static final PolicyFactory POLICY_SOME_HTML = new org.owasp.html.HtmlPolicyBuilder()\n        .allowElements(\"br\", \"p\", \"strong\", \"b\")\n        .toFactory();\nprivate static final PolicyFactory POLICY_NO_HTML = new org.owasp.html.HtmlPolicyBuilder().toFactory();\nprivate static final Pattern HTML_TAGS = Pattern.compile(\"<[a-z]+[^>]*>\");\n``` ",
+            "created_at": "2026-08-19T21:05:02Z"
+          },
+          {
+            "path": "misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java",
+            "line": 157,
+            "body": "**DOCUMENTATION**: Include `key` and `file` in this anchor-mismatch message. As written, a build processing many resource bundles cannot identify which property produced the diagnostic, whereas the nearby illegal-HTML message includes both values. \n The diagnostic lacks the context needed to locate and correct the offending translated property.\n```\nmessages.add(\"Didn't find anchor tag \" + matcher.group() + \" in original string for key \" + key + \" in file \" + file);\n``` ",
+            "created_at": "2026-08-19T21:05:03Z"
+          },
+          {
+            "path": "themes/src/main/resources-community/theme/base/login/messages/messages_lt.properties",
+            "line": 71,
+            "body": "**ERROR**: `messages_lt.properties` now assigns the Italian sentence `Installa una delle seguenti applicazioni sul tuo cellulare:` to `loginTotpStep1`. Restore an appropriate Lithuanian translation so Lithuanian users do not see Italian during TOTP setup. \n The changed value is in the wrong language for this locale and conflicts with the surrounding Lithuanian instructions.\n ",
+            "created_at": "2026-08-19T21:05:04Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 02f48f776f43734d1ac8914d3ad6a7115acdcb33.",
+            "created_at": "2026-08-19T21:05:05Z"
+          }
+        ]
       }
     ]
   },
@@ -3247,6 +3278,49 @@
             "created_at": "2026-06-28T21:20:12Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "keycloak__keycloak__deputydev-v5__PR37634__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR37634__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "server-spi-private/src/main/java/org/keycloak/protocol/oidc/grants/OAuth2GrantTypeFactory.java",
+            "line": 30,
+            "body": "**DOCUMENTATION**: Document this as an exactly two-character shortcut rather than a three-letter shortcut. The token-context decoder expects a six-character prefix made from three two-character segments, so following the current Javadoc would produce an incompatible grant shortcut. \n The documented shortcut length contradicts the two-character format enforced by token ID decoding.\n```\n/**\n * @return a unique 2-character shortcut for the specific grant\n */\nString getShortcut();\n``` ",
+            "created_at": "2026-08-19T21:10:44Z"
+          },
+          {
+            "path": "services/src/main/java/org/keycloak/protocol/oidc/encode/AccessTokenContext.java",
+            "line": 73,
+            "body": "**SECURITY**: This repeats the `grantType` null check while the message refers to `rawTokenId`. Validate `rawTokenId` here instead so a context cannot be constructed with that required field unset. \n The constructor currently validates grantType twice and stores rawTokenId without validating it.\n```\nObjects.requireNonNull(rawTokenId, \"Null rawTokenId not allowed\");\n``` ",
+            "created_at": "2026-08-19T21:10:45Z"
+          },
+          {
+            "path": "services/src/main/java/org/keycloak/protocol/oidc/encode/DefaultTokenContextEncoderProviderFactory.java",
+            "line": 108,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Both lazy lookup paths use a non-atomic `get`/null-check/discovery/`put` sequence. Concurrent misses for the same grant can therefore repeat `getProviderFactory` calls in `getShortcutByGrantType` or the full provider stream scan in `getGrantTypeByShortcut`. Use an atomic per-key loading operation for both caches. \n Concurrent callers can independently perform the same provider discovery before either caller populates the ConcurrentHashMap.\n ",
+            "created_at": "2026-08-19T21:10:47Z"
+          },
+          {
+            "path": "services/src/main/java/org/keycloak/protocol/oidc/encode/TokenContextEncoderProvider.java",
+            "line": 33,
+            "body": "**DOCUMENTATION**: Please add method-level Javadoc for these new provider contract methods, especially defining `rawTokenId` versus `encodedTokenId`, the returned context, and relevant failure behavior. The interface-level description does not specify how callers or implementations should use these values. \n All three newly introduced interface methods lack documentation of their inputs, outputs, and encoding contract.\n ",
+            "created_at": "2026-08-19T21:10:48Z"
+          },
+          {
+            "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AssertEvents.java",
+            "line": 483,
+            "body": "**SECURITY**: The grant shortcut is the final two characters of the six-character context prefix, so extract it with `substring(4, 6)`. The condition must also be negated: the matcher should return false when that segment does not equal `expectedGrantShortcut`, not when it does. \n The current matcher reads across the token-type/grant-type boundary and rejects equality, so it does not validate the expected grant shortcut.\n```\nif (!items[0].substring(4, 6).equals(expectedGrantShortcut)) return false;\n``` ",
+            "created_at": "2026-08-19T21:10:49Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit daffb05b5ad091d3ec09933c3c99c2b54f448696.",
+            "created_at": "2026-08-19T21:10:50Z"
+          }
+        ]
       }
     ]
   },
@@ -4705,6 +4779,31 @@
             "created_at": "2026-06-28T23:26:03Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "keycloak__keycloak__deputydev-v5__PR38446__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR38446__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java",
+            "line": 232,
+            "body": "**ERROR**: Guard the result of `getMyUser(user)` before accessing its fields. This map lookup can return null when the user is absent, so `myUser.recoveryCodes` currently throws a `NullPointerException`; return an empty credential stream in that case, as the other lookup methods already handle missing users. \n A missing entry in the internal users map makes getCredentials dereference null instead of returning no credentials.\n```\nvar myUser = getMyUser(user);\nif (myUser == null) {\n    return Stream.empty();\n}\nList<CredentialModel> credentialModels = new ArrayList<>();\n``` ",
+            "created_at": "2026-08-19T21:18:48Z"
+          },
+          {
+            "path": "testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java",
+            "line": 340,
+            "body": "**SECURITY**: Consume and persist a recovery code after a successful match. This branch only tests membership in the stored list, so the same code remains valid and can be replayed indefinitely. Match the local recovery-code provider's one-time-use behavior by removing the matched code before returning success. \n Successful federated recovery-code authentication leaves the code stored and reusable.\n```\nString challengeResponse = input.getChallengeResponse();\nboolean removed = generatedKeys.remove(challengeResponse);\nif (!removed) {\n    return false;\n}\ntry {\n    storedRecoveryKeys.setCredentialData(JsonSerialization.writeValueAsString(generatedKeys));\n} catch (IOException e) {\n    log.warnf(\"Cannot serialize updated recovery keys for user %s\", user.getUsername());\n    return false;\n}\nusers.get(translateUserName(user.getUsername())).recoveryCodes = generatedKeys.isEmpty() ? null : storedRecoveryKeys;\nreturn true;\n``` ",
+            "created_at": "2026-08-19T21:18:50Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit ce2dcc537b4b1f98a335e88cfe8e0b804483252a.",
+            "created_at": "2026-08-19T21:18:51Z"
+          }
+        ]
       }
     ]
   },
@@ -5863,6 +5962,31 @@
             "created_at": "2026-06-28T22:40:43Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "keycloak__keycloak__deputydev-v5__PR36882__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR36882__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/UpdateCommandDistTest.java",
+            "line": 49,
+            "body": "**ERROR**: Please also assert that this invocation exits with code 4. The test currently checks only the error text, so a regression that returns a different code would still pass even though automation relies on the documented feature-disabled exit code. \n The feature-disabled test does not protect the command's exit-code contract.\n```\ncliResult.assertError(\"Unable to use this command. The preview feature 'rolling-updates' is not enabled.\");\nassertEquals(4, cliResult.exitCode());\n``` ",
+            "created_at": "2026-08-19T20:45:44Z"
+          },
+          {
+            "path": "quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/UpdateCommandDistTest.java",
+            "line": 47,
+            "body": "**DOCUMENTATION**: Add a disabled-feature test for `UpdateCompatibilityCheck` as well. This test launches only the `metadata` sub-command, so removing or breaking the equivalent feature gate in the `check` sub-command would not be detected. \n Only the metadata path is exercised while the check command's equivalent disabled-feature path remains uncovered.\n```\n@Test\n@Launch({UpdateCompatibility.NAME, UpdateCompatibilityCheck.NAME})\npublic void testFeatureNotEnabledOnCheck(CLIResult cliResult) {\n    cliResult.assertError(\"Unable to use this command. The preview feature 'rolling-updates' is not enabled.\");\n    assertEquals(4, cliResult.exitCode());\n}\n``` ",
+            "created_at": "2026-08-19T20:45:45Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 62bf98939eda58f0d0a1a41a58aa6d4ecdc24dba.",
+            "created_at": "2026-08-19T20:45:46Z"
+          }
+        ]
       }
     ]
   },
@@ -7384,6 +7508,55 @@
             "created_at": "2026-06-28T22:50:30Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "keycloak__keycloak__deputydev-v5__PR36880__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR36880__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/AdminPermissions.java",
+            "line": 91,
+            "body": "**ERROR**: Avoid calling `clients().setPermissionsEnabled(client, false)` from this listener when V2 is active. The feature guard also passes in V2, `management(...)` then returns `MgmtPermissionsV2`, and `ClientPermissionsV2.setPermissionsEnabled` unconditionally throws `UnsupportedOperationException`. Consequently, deleting a client in V2 fails during event processing. Restrict this cleanup call to V1 or provide a V2-compatible deletion cleanup path. \n A client-removal event in V2 is dispatched to a method that explicitly throws, causing the deletion event handler to fail.\n ",
+            "created_at": "2026-08-19T20:52:36Z"
+          },
+          {
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java",
+            "line": 98,
+            "body": "**DOCUMENTATION**: Correct the Javadocs for `requireView()` and `requireViewClientScopes()`. Both methods return `void`, so saying they return `true` is inaccurate; like the other `require*` methods, their comments should describe throwing `ForbiddenException` when the corresponding check returns `false`. \n The new documentation describes boolean return values for two void methods and therefore misstates their contracts.\n```\n/**\n * Throws ForbiddenException if {@link #canView()} returns {@code false}.\n */\nvoid requireView();\n\n/**\n * Throws ForbiddenException if {@link #canViewClientScopes()} returns {@code false}.\n */\nvoid requireViewClientScopes();\n``` ",
+            "created_at": "2026-08-19T20:52:37Z"
+          },
+          {
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java",
+            "line": 193,
+            "body": "**ERROR**: The existing client management-permissions REST endpoints can route to this V2 implementation, but `isPermissionsEnabled`, `setPermissionsEnabled`, `resource`, and `getPermissions` all throw `UnsupportedOperationException`. GET or update requests to those endpoints therefore produce an unhandled 500 in V2. Prevent the V1 endpoints from selecting this implementation or provide meaningful V2 behavior for the methods they invoke. \n Public REST paths select MgmtPermissionsV2 and invoke methods that this class unconditionally rejects with UnsupportedOperationException.\n ",
+            "created_at": "2026-08-19T20:52:38Z"
+          },
+          {
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java",
+            "line": 262,
+            "body": "**CODE_MAINTAINABILTIY**: Remove the private `getEvaluationContext` method if it is not intended to participate in permission evaluation. It has no callers, and the actual `hasPermission` paths invoke `root.evaluatePermission(...)` directly, so this method is currently dead and obscures which evaluation context is used. \n A private method with no call sites cannot affect execution and misleadingly suggests that client-specific evaluation context is in use.\n```\n// Remove the unused private getEvaluationContext(ClientModel, AccessToken) method.\n``` ",
+            "created_at": "2026-08-19T20:52:39Z"
+          },
+          {
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java",
+            "line": 46,
+            "body": "**CODE_MAINTAINABILTIY**: Remove the unused static import of `AdminPermissionManagement.TOKEN_EXCHANGE`; the constant is never referenced in this class. \n The imported constant has no use in the new implementation.\n```\n// Remove:\nimport static org.keycloak.services.resources.admin.permissions.AdminPermissionManagement.TOKEN_EXCHANGE;\n``` ",
+            "created_at": "2026-08-19T20:52:39Z"
+          },
+          {
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java",
+            "line": 225,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Both `hasPermission` overloads allocate `Arrays.asList(scope)` only to test membership in a single-element list. Compare directly with `scope.equals(s)`, as `hasGrantedPermission` already does, to remove the unnecessary allocation and simplify the duplicated loops. \n Every invocation of either permission helper creates a list for an operation that is semantically a single string equality check.\n```\nfor (Permission permission : permissions) {\n    for (String s : permission.getScopes()) {\n        if (scope.equals(s)) {\n            return true;\n        }\n    }\n}\n``` ",
+            "created_at": "2026-08-19T20:52:40Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 1950a511026d520a7329c7b6b9ee60a4af8f8b55.",
+            "created_at": "2026-08-19T20:52:41Z"
+          }
+        ]
       }
     ]
   },
@@ -8963,6 +9136,43 @@
             "created_at": "2026-06-28T22:57:21Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "keycloak__keycloak__deputydev-v5__PR37038__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR37038__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionEvaluator.java",
+            "line": 133,
+            "body": "**DOCUMENTATION**: The `requireManageMembers` method in `GroupPermissionEvaluator` has a copy-pasted Javadoc that incorrectly references `canManageMembership(GroupModel)` instead of the semantically accurate `canManageMembers(GroupModel)`. This is misleading to API consumers and implementors trying to understand which check is enforced. \n The Javadoc for `requireManageMembers` describes the check performed by the preceding `requireManageMembership` method, giving this method an incorrect documented contract.\n```\n/**\n     * Throws ForbiddenException if {@link #canManageMembers(GroupModel)} returns {@code false}.\n     */\n    void requireManageMembers(GroupModel group);\n``` ",
+            "created_at": "2026-08-19T20:59:08Z"
+          },
+          {
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionEvaluator.java",
+            "line": 142,
+            "body": "**DOCUMENTATION**: The `@return` documentation says this method returns a stream, but its declared return type is `Set<String>`. Update the Javadoc to describe a set of group IDs so callers are not given the wrong API contract. \n The Javadoc's stated return type directly contradicts the `Set<String>` method signature.\n```\n/**\n     * If {@link UserPermissionEvaluator#canView()} evaluates to {@code true}, returns an empty set.\n     *\n     * @return set of IDs of groups with view permission\n     */\n    Set<String> getGroupIdsWithViewPermission();\n``` ",
+            "created_at": "2026-08-19T20:59:09Z"
+          },
+          {
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java",
+            "line": 70,
+            "body": "**SECURITY**: `canManage()` passes both `VIEW` and `MANAGE` to `hasPermission`, whose loop returns true when any requested scope is granted. Consequently, a caller with only the all-groups `VIEW` scope passes this manage gate. Require `AdminPermissionsSchema.MANAGE` here, consistent with the group-specific manage check. \n A VIEW-only all-groups permission is treated as sufficient for a manage authorization decision.\n```\n@Override\npublic boolean canManage() {\n    if (root.hasOneAdminRole(AdminRoles.MANAGE_USERS)) {\n        return true;\n    }\n\n    return hasPermission(null, AdminPermissionsSchema.MANAGE);\n}\n``` ",
+            "created_at": "2026-08-19T20:59:09Z"
+          },
+          {
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java",
+            "line": 122,
+            "body": "**SECURITY**: The callback uses `groupResource.getId()` for both `hasPermission` and the returned `granted` set, but that value is the authorization resource UUID. The group model ID is stored in `groupResource.getName()`. Passing the resource UUID to `hasPermission` makes its `findByName` lookup miss the group-specific resource and fall back to all-groups evaluation, while returning that UUID also gives user filtering the wrong identifier. Use `getName()` in both places. \n Confusing the authorization resource UUID with the group model ID causes the wrong permission resource to be evaluated and produces invalid group IDs for provider-side user filtering.\n```\nresourceStore.findByType(server, AdminPermissionsSchema.GROUPS_RESOURCE_TYPE, groupResource -> {\n    String groupModelId = groupResource.getName();\n    if (hasPermission(groupModelId, AdminPermissionsSchema.VIEW_MEMBERS, AdminPermissionsSchema.MANAGE_MEMBERS)) {\n        granted.add(groupModelId);\n    }\n});\n``` ",
+            "created_at": "2026-08-19T20:59:10Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 7355f05e6aafe2a7acc6434fb40c686665416606.",
+            "created_at": "2026-08-19T20:59:11Z"
+          }
+        ]
       }
     ]
   },
@@ -10427,6 +10637,43 @@
             "created_at": "2026-06-28T22:36:15Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "keycloak__keycloak__deputydev-v5__PR33832__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR33832__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "authz/client/pom.xml",
+            "line": 68,
+            "body": "**SECURITY**: Remove the added Hamcrest dependency: the module's test uses `org.junit.Assert.assertArrayEquals` and the supplied search found no Hamcrest imports or matcher usage. \n The explicit test dependency is unused by the module's test code and adds unnecessary build configuration.\n```\n<!-- Remove this unused dependency:\n<dependency>\n    <groupId>org.hamcrest</groupId>\n    <artifactId>hamcrest</artifactId>\n    <scope>test</scope>\n</dependency>\n-->\n``` ",
+            "created_at": "2026-08-19T20:40:13Z"
+          },
+          {
+            "path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/ASN1Decoder.java",
+            "line": 55,
+            "body": "**CODE_MAINTAINABILTIY**: Reject an indefinite length here before entering the loop. `readLength()` returns `-1` for that encoding, so `while (length > 0)` currently skips parsing and returns an empty sequence instead of reporting the unsupported encoding. \n Malformed indefinite-length input is represented as an empty sequence and only fails later with the misleading sequence-size error.\n```\nint length = readLength();\nif (length < 0) {\n    throw new IOException(\"Indefinite-length encoding is not supported in DER\");\n}\n``` ",
+            "created_at": "2026-08-19T20:40:14Z"
+          },
+          {
+            "path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/AuthzClientCryptoProvider.java",
+            "line": 61,
+            "body": "**ERROR**: `getBouncyCastleProvider()` currently returns the provider backing the JDK's default keystore type, not a BouncyCastle provider as its contract states. Return an actual BouncyCastle provider, or explicitly signal that BouncyCastle is unsupported, rather than exposing an unrelated provider to provider-specific algorithm lookups. \n Callers expecting BouncyCastle can receive a JDK keystore provider and fail when requesting algorithms from the returned provider.\n```\n@Override\npublic Provider getBouncyCastleProvider() {\n    return null;\n}\n``` ",
+            "created_at": "2026-08-19T20:40:15Z"
+          },
+          {
+            "path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/AuthzClientCryptoProvider.java",
+            "line": 114,
+            "body": "**CODE_MAINTAINABILTIY**: Remove these two standalone `ASN1Encoder.create().write(...)` calls. Their results are discarded, and the return expression immediately performs the same encodings again when constructing the DER sequence. \n The statements have no effect on the output and perform redundant allocations and DER encoding work on every invocation.\n```\nreturn ASN1Encoder.create()\n        .writeDerSeq(\n                ASN1Encoder.create().write(rBigInteger),\n                ASN1Encoder.create().write(sBigInteger))\n        .toByteArray();\n``` ",
+            "created_at": "2026-08-19T20:40:15Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 79d11c4890cc4792e3dc3535af59e4e5f7a2739d.",
+            "created_at": "2026-08-19T20:40:16Z"
+          }
+        ]
       }
     ]
   },
@@ -11653,6 +11900,37 @@
             "created_at": "2026-06-28T23:30:25Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "keycloak__keycloak__deputydev-v5__PR40940__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR40940__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/GroupAdapter.java",
+            "line": 275,
+            "body": "**ERROR**: Do not return `null` here: the `GroupModel.getSubGroupsCount()` contract explicitly states that it never returns null. This implementation can therefore propagate an unexpected null to representations or fail when a caller unboxes the result. Use a non-null fallback such as `0L` when the supplied model is absent. \n The new null branch violates the documented interface contract and exposes callers to null propagation or unboxing failures.\n```\n@Override\npublic Long getSubGroupsCount() {\n    if (isUpdated()) return updated.getSubGroupsCount();\n    GroupModel model = modelSupplier.get();\n    return model == null ? 0L : model.getSubGroupsCount();\n}\n``` ",
+            "created_at": "2026-08-19T21:24:15Z"
+          },
+          {
+            "path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/GroupAdapter.java",
+            "line": 274,
+            "body": "**PERFORMANCE_OPTIMIZATION**: This guard fixes only `getSubGroupsCount()`. The three adjacent `getSubGroupsStream(...)` overloads still directly dereference `modelSupplier.get()`, so they can encounter the same null result during concurrent deletion and throw an NPE. Apply an equivalent guard to those overloads and return an empty stream when the model is absent. \n The concurrent-deletion null handling remains incomplete across sibling subgroup access methods.\n```\n@Override\npublic Stream<GroupModel> getSubGroupsStream(String search, Integer firstResult, Integer maxResults) {\n    if (isUpdated()) return updated.getSubGroupsStream(search, firstResult, maxResults);\n    GroupModel model = modelSupplier.get();\n    return model == null ? Stream.empty() : model.getSubGroupsStream(search, firstResult, maxResults);\n}\n\n@Override\npublic Stream<GroupModel> getSubGroupsStream(Integer firstResult, Integer maxResults) {\n    if (isUpdated()) return updated.getSubGroupsStream(firstResult, maxResults);\n    GroupModel model = modelSupplier.get();\n    return model == null ? Stream.empty() : model.getSubGroupsStream(firstResult, maxResults);\n}\n\n@Override\npublic Stream<GroupModel> getSubGroupsStream(String search, Boolean exact, Integer firstResult, Integer maxResults) {\n    if (isUpdated()) return updated.getSubGroupsStream(search, exact, firstResult, maxResults);\n    GroupModel model = modelSupplier.get();\n    return model == null ? Stream.empty() : model.getSubGroupsStream(search, exact, firstResult, maxResults);\n}\n``` ",
+            "created_at": "2026-08-19T21:24:16Z"
+          },
+          {
+            "path": "tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java",
+            "line": 155,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Retain and join the reader thread before checking `caughtExceptions`. After this flag is set, the reader may still be inside `groups()` and append an exception after the current assertion has already passed. Also synchronize or record that at least one read completed so the deletion loop cannot finish without exercising the concurrent-read path. \n Missing thread synchronization permits false passes from late exceptions and does not guarantee that the regression path ran.\n```\nAtomicBoolean readerRanAtLeastOnce = new AtomicBoolean(false);\nThread readerThread = new Thread(() -> {\n    while (!deletedAll.get()) {\n        try {\n            managedRealm.admin().groups().groups(null, 0, Integer.MAX_VALUE, true);\n            readerRanAtLeastOnce.set(true);\n        } catch (Exception e) {\n            caughtExceptions.add(e);\n        }\n    }\n});\nreaderThread.start();\n\n// delete groups\ngroupUuuids.forEach(groupUuid ->\n        managedRealm.admin().groups().group(groupUuid).remove());\ndeletedAll.set(true);\nreaderThread.join(5_000);\n\nassertTrue(readerRanAtLeastOnce.get(), \"Reader thread never ran during deletions\");\nassertThat(caughtExceptions, Matchers.empty());\n``` ",
+            "created_at": "2026-08-19T21:24:17Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 4def149a50c4b9b2c24fbb753ac60fe4f4df0cac.",
+            "created_at": "2026-08-19T21:24:18Z"
+          }
+        ]
       }
     ]
   },
@@ -13015,6 +13293,25 @@
             "created_at": "2026-06-28T22:13:01Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "keycloak__keycloak-greptile__deputydev-v5__PR1__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak-greptile__deputydev-v5__PR1__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "services/src/main/java/org/keycloak/authentication/authenticators/browser/UsernameForm.java",
+            "line": 47,
+            "body": "**CODE_MAINTAINABILTIY**: This call does not compile: the only available method is `isConditionalPasskeysEnabled(UserModel user)`, so there is no zero-argument overload. Pass the already-checked user as `isConditionalPasskeysEnabled(context.getUser())`. \n Java cannot resolve the newly added zero-argument call because the evidenced inherited method requires a UserModel argument.\n```\nif (context.getUser() != null && !isConditionalPasskeysEnabled(context.getUser())) {\n``` ",
+            "created_at": "2026-08-19T20:26:35Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 3214b188de808fdb0fff335556b686f1b0a63218.",
+            "created_at": "2026-08-19T20:26:37Z"
+          }
+        ]
       }
     ]
   },
@@ -14441,6 +14738,55 @@
             "created_at": "2026-06-28T23:04:21Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "sentry__sentry__deputydev-v5__PR93824__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/sentry__sentry__deputydev-v5__PR93824__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "src/sentry/spans/consumers/process/flusher.py",
+            "line": 247,
+            "body": "**ERROR**: Use `>= MAX_PROCESS_RESTARTS` here. With the current `>` check, a worker whose restart count already equals the maximum is restarted once more, and the error is raised only after the following failure. \n The guard runs before the counter increment, so equality currently permits `MAX_PROCESS_RESTARTS + 1` restarts.\n```\nif self.process_restarts[process_index] >= MAX_PROCESS_RESTARTS:\n    raise RuntimeError(\n        f\"flusher process for shards {shards} crashed repeatedly ({cause}), restarting consumer\"\n    )\nself.process_restarts[process_index] += 1\n``` ",
+            "created_at": "2026-08-19T22:46:30Z"
+          },
+          {
+            "path": "src/sentry/spans/consumers/process/flusher.py",
+            "line": 31,
+            "body": "**DOCUMENTATION**: The docstring should not say this creates one process per shard. `num_processes` is capped by `max_processes`, and the round-robin mapping assigns multiple shards to each worker whenever the process limit is lower than the shard count. \n The documentation contradicts the implemented capped, shared-shard worker model.\n```\nclass SpanFlusher(ProcessingStrategy[FilteredPayload | int]):\n    \"\"\"\n    A background multiprocessing manager that polls Redis for new segments to flush and to produce to Kafka.\n    Distributes assigned shards across up to ``max_processes`` workers; each worker handles one or more\n    shards depending on how many shards are assigned relative to the process limit.\n    \"\"\"\n``` ",
+            "created_at": "2026-08-19T22:46:31Z"
+          },
+          {
+            "path": "src/sentry/spans/consumers/process/flusher.py",
+            "line": 199,
+            "body": "**CODE_MAINTAINABILTIY**: Use the `shard` tag key here, matching the neighboring `flusher.produce` and `segment_size_bytes` metrics. The current `shards` key creates a separate dimension, so queries and dashboards filtering on `shard` will omit this timer. \n Related metrics attach the same `shard_tag` value under inconsistent tag keys.\n```\nwith metrics.timer(\"spans.buffer.flusher.wait_produce\", tags={\"shard\": shard_tag}):\n``` ",
+            "created_at": "2026-08-19T22:46:32Z"
+          },
+          {
+            "path": "src/sentry/spans/consumers/process/flusher.py",
+            "line": 51,
+            "body": "**SECURITY**: Reject negative `max_processes` values before constructing the shard mapping. The CLI accepts any integer, and a negative value remains truthy here, making `num_processes` negative and leaving no valid process entries for the subsequent shard assignment. \n An unconstrained negative process limit prevents construction of a valid worker-to-shard mapping.\n```\nif max_processes is not None and max_processes < 1:\n    raise ValueError(f\"max_processes must be >= 1, got {max_processes}\")\nself.max_processes = max_processes or len(buffer.assigned_shards)\n``` ",
+            "created_at": "2026-08-19T22:46:33Z"
+          },
+          {
+            "path": "src/sentry/spans/consumers/process/flusher.py",
+            "line": 254,
+            "body": "**ERROR**: Handle the thread-worker restart path before creating a replacement. When `produce_to_pipe` is set, the worker is a `threading.Thread`; this branch does not stop it, but line +259 still starts another worker for the same shards. A hung thread can therefore remain alive across repeated restarts. \n Restart cleanup only handles multiprocessing workers while replacement applies to both processes and threads.\n ",
+            "created_at": "2026-08-19T22:46:34Z"
+          },
+          {
+            "path": "src/sentry/spans/consumers/process/flusher.py",
+            "line": 235,
+            "body": "**DOCUMENTATION**: This comment should describe a process-level hang check, not an individual-shard health check. The condition reads one `process_healthy_since[process_index]` timestamp and marks the entire worker as hung; it does not inspect shard-specific health. \n The comment claims per-shard health evaluation that the code does not perform.\n```\n# The process has not updated its healthy_since timestamp within\n# the allowed window, so consider the entire process hung.\ncause = \"hang\"\n``` ",
+            "created_at": "2026-08-19T22:46:35Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 3162ad68a5c87666788b27a44eb31235025091a9.",
+            "created_at": "2026-08-19T22:46:36Z"
+          }
+        ]
       }
     ]
   },
@@ -16412,6 +16758,37 @@
             "created_at": "2026-06-28T22:58:37Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "sentry__sentry-greptile__deputydev-v5__PR5__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/sentry__sentry-greptile__deputydev-v5__PR5__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "src/sentry/issues/endpoints/browser_reporting_collector.py",
+            "line": 52,
+            "body": "**ERROR**: Use a presence check rather than truthiness when enforcing mutual exclusivity. With `self.initial_data.get(\"age\")`, an `age` value of `0` is falsy, so a payload containing both `age=0` and `timestamp` passes validation. The same bypass exists for `timestamp=0` in `validate_age`. \n Zero-valued fields bypass the intended age/timestamp mutual-exclusivity checks.\n```\ndef validate_timestamp(self, value: int) -> int:\n        if \"age\" in self.initial_data and self.initial_data[\"age\"] is not None:\n            raise serializers.ValidationError(\"If timestamp is present, age must be absent\")\n        return value\n\n    def validate_age(self, value: int) -> int:\n        if \"timestamp\" in self.initial_data and self.initial_data[\"timestamp\"] is not None:\n            raise serializers.ValidationError(\"If age is present, timestamp must be absent\")\n        return value\n``` ",
+            "created_at": "2026-08-19T22:41:10Z"
+          },
+          {
+            "path": "src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py",
+            "line": 118,
+            "body": "**ERROR**: Do not pair `error_ids` with `events.values()` by position. The dictionary returned by `get_multi` may not iterate in request order, and omitted entries shift every subsequent pairing. Retain a `node_id` to `event_id` mapping and associate each returned item by its key. \n A reordered or incomplete nodestore result can attach an event's title, message, and timestamp to the wrong error ID.\n```\nnode_ids = [Event.generate_node_id(project_id, event_id=id) for id in error_ids]\n        node_id_to_event_id = dict(zip(node_ids, error_ids))\n        events = nodestore.backend.get_multi(node_ids)\n\n        return [\n            ErrorEvent(\n                category=\"error\",\n                id=node_id_to_event_id[node_id],\n                title=data.get(\"title\", \"\"),\n                timestamp=data.get(\"timestamp\", 0.0),\n                message=data.get(\"message\", \"\"),\n            )\n            for node_id, data in events.items()\n            if data is not None\n        ]\n``` ",
+            "created_at": "2026-08-19T22:41:11Z"
+          },
+          {
+            "path": "static/app/views/dashboards/widgetCard/chart.tsx",
+            "line": 164,
+            "body": "**ERROR**: The feature-enabled branch always passes empty `columns`, `data`, `fields`, and `units` to `TableWidgetVisualization`. Consequently, organizations with `use-table-widget-visualization` enabled render a blank table regardless of `result.data` and `result.meta`. Build the visualization props from the actual query result instead of these placeholders. \n The feature gate discards all available table results and unconditionally renders empty content.\n ",
+            "created_at": "2026-08-19T22:41:12Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9.",
+            "created_at": "2026-08-19T22:41:13Z"
+          }
+        ]
       }
     ]
   },
@@ -18067,6 +18444,49 @@
             "created_at": "2026-06-28T22:12:55Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "sentry__sentry-greptile__deputydev-v5__PR1__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/sentry__sentry-greptile__deputydev-v5__PR1__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "src/sentry/api/endpoints/organization_auditlogs.py",
+            "line": 73,
+            "body": "**CODE_MAINTAINABILTIY**: Both branches repeat the same `self.paginate()` arguments and differ only in `paginator_cls` and `enable_advanced_features`. Select those conditional values first and keep a single paginate call so future changes to the shared arguments cannot diverge between paths. \n The duplicated request, queryset, ordering, and serialization arguments create two locations that must remain synchronized.\n ",
+            "created_at": "2026-08-19T22:18:51Z"
+          },
+          {
+            "path": "src/sentry/api/paginator.py",
+            "line": 840,
+            "body": "**ERROR**: `get_item_key` passes the ordering value directly to `math.floor` or `math.ceil`, but the audit-log endpoint instantiates this paginator with `order_by=\"-datetime\"`. That makes `value` a `datetime`, so cursor generation raises `TypeError` on every optimized response. Convert datetime keys to the same numeric representation used by the existing datetime paginator before applying numeric rounding. \n `math.floor` and `math.ceil` require numeric values, while the configured audit-log cursor key is a datetime object.\n```\ndef get_item_key(self, item, for_prev=False):\n    value = getattr(item, self.key)\n    if isinstance(value, datetime):\n        import calendar\n        value = calendar.timegm(value.timetuple()) * 1000\n    return int(math.floor(value) if self._is_asc(for_prev) else math.ceil(value))\n``` ",
+            "created_at": "2026-08-19T22:18:52Z"
+          },
+          {
+            "path": "src/sentry/api/paginator.py",
+            "line": 882,
+            "body": "**ERROR**: This branch is entered specifically when `cursor.offset < 0`, assigns that negative value to `start_offset`, and then uses it as the start of a Django QuerySet slice. Django does not support negative QuerySet indices, so an optimized request with such a cursor raises an unhandled exception instead of returning a page. Permission scoping does not make the slice valid; reject, clamp, or redesign negative-offset pagination before enabling this path. \n The advanced branch deterministically passes a negative start index to `queryset[start_offset:stop]`, which Django rejects.\n```\nstart_offset = max(0, offset)\nstop = start_offset + limit + extra\nresults = list(queryset[start_offset:stop])\n``` ",
+            "created_at": "2026-08-19T22:18:53Z"
+          },
+          {
+            "path": "src/sentry/api/paginator.py",
+            "line": 179,
+            "body": "**DOCUMENTATION**: The comment says negative offsets are safe because the underlying queryset handles boundary conditions, but Django QuerySets reject negative slice indices. Remove this safety claim or document the actual non-negative offset invariant. \n The new comment describes unsupported negative QuerySet slicing as safe and therefore obscures the required offset constraint.\n```\n# QuerySet slicing requires a non-negative start offset.\nstart_offset = max(0, offset) if not cursor.is_prev else offset\n``` ",
+            "created_at": "2026-08-19T22:18:54Z"
+          },
+          {
+            "path": "src/sentry/api/paginator.py",
+            "line": 821,
+            "body": "**PERFORMANCE_OPTIMIZATION**: `OptimizedCursorPaginator.get_result` duplicates the base implementation's cursor initialization, hit counting, boundary trimming, reversal, cursor construction, and post-query filtering. The only distinct behavior is offset selection, so future fixes to the base paginator will not automatically reach this class. Extract a narrow offset-resolution hook or otherwise delegate the shared flow to `BasePaginator` instead of copying the complete method. \n Two independent implementations of the same pagination flow can diverge whenever one receives a bug fix or cursor-handling change.\n ",
+            "created_at": "2026-08-19T22:18:55Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 8ab88145113dd23a930e23b9cbbcf8b30e4c0b17.",
+            "created_at": "2026-08-19T22:18:56Z"
+          }
+        ]
       }
     ]
   },
@@ -19332,6 +19752,25 @@
             "created_at": "2026-06-29T00:07:24Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "grafana__grafana__deputydev-v5__PR97529__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR97529__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "pkg/storage/unified/resource/server.go",
+            "line": 258,
+            "body": "**DOCUMENTATION**: The duplicate error log in `NewResourceServer` is redundant with the existing error log inside `server.Init`. Both log \"error initializing resource server\" with the same error, producing two identical log lines for a single failure event. \n Inside `server.Init`, the code already calls `s.log.Error(\"error initializing resource server\", \"error\", s.initErr)` before returning. The new constructor-level log reports that same failure again, producing duplicate entries.\n```\nerr := s.Init(ctx)\nif err != nil {\n\treturn nil, err\n}\n``` ",
+            "created_at": "2026-08-20T00:55:23Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 26fed312840cd76b766cbd2158e17a7e6c0ec548.",
+            "created_at": "2026-08-20T00:55:24Z"
+          }
+        ]
       }
     ]
   },
@@ -20621,6 +21060,31 @@
             "created_at": "2026-06-28T23:46:01Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "sentry__sentry__deputydev-v5__PR80168__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/sentry__sentry__deputydev-v5__PR80168__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "src/sentry/incidents/grouptype.py",
+            "line": 12,
+            "body": "**CODE_MAINTAINABILTIY**: `MetricAlertDetectorHandler` now extends `StatefulDetectorHandler` but implements none of its abstract hooks, including the newly added `build_occurrence_and_event_data`. Consequently, Python treats this handler as abstract and raises `TypeError` when it is instantiated. Implement the required stateful-handler methods before switching to this base class, or retain the previous base until the implementation is available. \n An ABC subclass that does not implement its abstract methods cannot be instantiated, blocking any path that resolves this detector handler.\n ",
+            "created_at": "2026-08-19T23:08:29Z"
+          },
+          {
+            "path": "src/sentry/workflow_engine/processors/detector.py",
+            "line": 226,
+            "body": "**DOCUMENTATION**: Update this docstring to match the new return type. `evaluate` now returns a `dict[DetectorGroupKey, DetectorEvaluationResult]`, but the text still says it returns a list. \n The docstring directly contradicts the method's annotation and dictionary-based implementation.\n```\ndef evaluate(\n    self, data_packet: DataPacket[T]\n) -> dict[DetectorGroupKey, DetectorEvaluationResult]:\n    \"\"\"\n    Evaluates a data packet and returns DetectorEvaluationResult values keyed by\n    DetectorGroupKey.\n    \"\"\"\n``` ",
+            "created_at": "2026-08-19T23:08:30Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 8422030ef456e3a898415e96475b4d8ddfc7640f.",
+            "created_at": "2026-08-19T23:08:31Z"
+          }
+        ]
       }
     ]
   },
@@ -21792,6 +22256,37 @@
             "created_at": "2026-06-28T23:21:58Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "sentry__sentry__deputydev-v5__PR80528__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/sentry__sentry__deputydev-v5__PR80528__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "src/sentry/monitors/logic/incident_occurrence.py",
+            "line": 46,
+            "body": "**PERFORMANCE_OPTIMIZATION**: `create_incident_occurrence` is called for every check-in in `incidents.py`, so this fetch repeats for the same `monitor_env` on every iteration. Fetch the last successful check-in once before that loop and pass the precomputed result into this function. \n An incident containing multiple check-ins performs the same last-successful-check-in lookup once per occurrence instead of once per monitor environment.\n```\n# In incidents.py, fetch once before the loop:\nlast_successful_checkin = monitor_env.get_last_successful_checkin()\nfor checkin in checkins:\n    create_incident_occurrence(\n        previous_checkins,\n        checkin,\n        incident,\n        received=received,\n        last_successful_checkin=last_successful_checkin,\n    )\n``` ",
+            "created_at": "2026-08-19T22:56:37Z"
+          },
+          {
+            "path": "src/sentry/monitors/logic/incident_occurrence.py",
+            "line": 168,
+            "body": "**ERROR**: Return the local `config` here. The function copies the monitor config and replaces `schedule_type` with its display value, but returning `monitor_environment.monitor.config` discards that transformation and exposes the original value. \n The transformed configuration created at lines 160-162 is never included in the returned monitor context.\n```\n\"config\": config,\n``` ",
+            "created_at": "2026-08-19T22:56:38Z"
+          },
+          {
+            "path": "src/sentry/monitors/logic/incident_occurrence.py",
+            "line": 145,
+            "body": "**ERROR**: Handle a total of zero before the singular and multi-status paths. If the input is empty or every status is excluded by `HUMAN_FAILURE_STATUS_MAP`, `get_text_list([])` produces an empty value and the function returns the malformed user-facing string `\" check-ins detected\"` instead of the documented fallback. \n The function has no branch for an empty `status_counts`, so its fallback formatting produces a failure reason with no subject.\n```\ntotal = sum(status_counts.values())\nif total == 0:\n    return _(\"A failed check-in was detected\")\nif total == 1:\n    return SINGULAR_HUMAN_FAILURE_MAP[list(status_counts.keys())[0]]\n``` ",
+            "created_at": "2026-08-19T22:56:39Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit dcdcadb771128e79259cc9eff9c70c38fc597976.",
+            "created_at": "2026-08-19T22:56:40Z"
+          }
+        ]
       }
     ]
   },
@@ -23038,6 +23533,37 @@
             "created_at": "2026-06-28T23:18:05Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "sentry__sentry__deputydev-v5__PR77754__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/sentry__sentry__deputydev-v5__PR77754__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "src/sentry/integrations/services/assignment_source.py",
+            "line": 18,
+            "body": "**ERROR**: `timezone.now()` is evaluated once when this class is defined, so every instance that omits `queued` receives the module-import timestamp. Use `field(default_factory=timezone.now)` to generate a fresh timestamp for each `AssignmentSource`. \n Instances created at different times otherwise receive the same stale `queued` value.\n```\nfrom dataclasses import asdict, dataclass, field\n\n@dataclass(frozen=True)\nclass AssignmentSource:\n    source_name: str\n    integration_id: int\n    queued: datetime = field(default_factory=timezone.now)\n``` ",
+            "created_at": "2026-08-19T22:51:03Z"
+          },
+          {
+            "path": "tests/sentry/integrations/services/test_assignment_source.py",
+            "line": 8,
+            "body": "**CODE_MAINTAINABILTIY**: This test passes an empty dictionary, not an array. Rename it to `test_from_dict_empty_dict` so the test name accurately describes its input. \n The current name incorrectly labels the tested dictionary as an array.\n```\ndef test_from_dict_empty_dict(self):\n    data: dict[str, Any] = {}\n    result = AssignmentSource.from_dict(data)\n    assert result is None\n``` ",
+            "created_at": "2026-08-19T22:51:04Z"
+          },
+          {
+            "path": "tests/sentry/integrations/services/test_assignment_source.py",
+            "line": 13,
+            "body": "**CODE_MAINTAINABILTIY**: Rename `test_from_dict_inalid_data` to `test_from_dict_invalid_data` to correct the typo in the test name. \n The misspelled test identifier reduces readability and appears verbatim in test reports.\n```\ndef test_from_dict_invalid_data(self):\n``` ",
+            "created_at": "2026-08-19T22:51:05Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 9501091c52ae94e8d916f79b35d21975b3f9cadb.",
+            "created_at": "2026-08-19T22:51:06Z"
+          }
+        ]
       }
     ]
   },
@@ -24429,6 +24955,43 @@
             "created_at": "2026-06-28T23:34:19Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "sentry__sentry__deputydev-v5__PR95633__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/sentry__sentry__deputydev-v5__PR95633__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py",
+            "line": 148,
+            "body": "**ERROR**: Only mark the offset complete when `result_processor` succeeds. The exception handler swallows processing failures, but this `finally` block still calls `complete_offset`, so the commit loop can commit the failed message and prevent it from being redelivered. This also contradicts the strategy's documented guarantee that offsets are committed only after successful processing. \n A processor exception is logged and then the failed offset is unconditionally marked complete, making it eligible for commit.\n```\nsuccess = False\ntry:\n    with sentry_sdk.start_transaction(\n        op=\"queue_worker.process\",\n        name=f\"monitors.{self.identifier}.worker_{self.worker_id}\",\n    ):\n        self.result_processor(self.identifier, work_item.result)\n    success = True\nexcept Exception:\n    logger.exception(\n        \"Unexpected error in queue worker\", extra={\"worker_id\": self.worker_id}\n    )\nfinally:\n    if success:\n        self.offset_tracker.complete_offset(work_item.partition, work_item.offset)\n``` ",
+            "created_at": "2026-08-19T23:03:44Z"
+          },
+          {
+            "path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py",
+            "line": 185,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Use a bounded queue here so submission applies backpressure when workers fall behind. `queue.Queue()` has no capacity limit, and every queued `WorkItem` retains its full message, allowing a sustained processing backlog to grow memory usage without bound. \n Unbounded queues retain every submitted message when producers outpace workers, creating unbounded heap growth.\n```\nwork_queue: queue.Queue[WorkItem[T]] = queue.Queue(maxsize=100)  # tune as needed\n``` ",
+            "created_at": "2026-08-19T23:03:44Z"
+          },
+          {
+            "path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py",
+            "line": 233,
+            "body": "**ERROR**: Do not set `worker.shutdown` before signaling the queues to shut down gracefully. After finishing its current item, each worker checks `while not self.shutdown` and exits without dequeuing the remaining backlog, defeating `shutdown(immediate=False)` and leaving those offsets outstanding. \n The early shutdown flag causes workers to stop between items instead of draining already queued work.\n```\ndef shutdown(self) -> None:\n    \"\"\"Gracefully shutdown all workers, draining queues first.\"\"\"\n    for q in self.queues:\n        try:\n            q.shutdown(immediate=False)\n        except Exception:\n            logger.exception(\"Error shutting down queue\")\n\n    for worker in self.workers:\n        worker.join(timeout=5.0)\n\n    for worker in self.workers:\n        worker.shutdown = True\n``` ",
+            "created_at": "2026-08-19T23:03:45Z"
+          },
+          {
+            "path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py",
+            "line": 145,
+            "body": "**DOCUMENTATION**: Include `work_item.partition` and `work_item.offset` in this exception log. Logging only the worker ID does not identify which Kafka message failed, even though both identifiers are available here. \n Processing failures cannot be tied to the responsible partition and offset from the emitted log context.\n```\nlogger.exception(\n    \"Unexpected error in queue worker\",\n    extra={\n        \"worker_id\": self.worker_id,\n        \"partition\": work_item.partition,\n        \"offset\": work_item.offset,\n    },\n)\n``` ",
+            "created_at": "2026-08-19T23:03:46Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 9966ec5a13e331659c3ea00981f9b11b0faf821f.",
+            "created_at": "2026-08-19T23:03:48Z"
+          }
+        ]
       }
     ]
   },
@@ -26196,6 +26759,43 @@
             "created_at": "2026-06-28T22:21:53Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "sentry__sentry-greptile__deputydev-v5__PR2__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/sentry__sentry-greptile__deputydev-v5__PR2__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "src/sentry/api/paginator.py",
+            "line": 821,
+            "body": "**CODE_MAINTAINABILTIY**: `OptimizedCursorPaginator.get_result` duplicates the existing paginator flow for cursor initialization, hit counting, slicing, previous-page handling, cursor construction, and post-query filtering. Changes or fixes to the base implementation can therefore diverge silently. Please factor the shared flow into the existing inheritance path and override only the offset behavior that differs. \n The new method copies the base pagination algorithm, creating two implementations that must remain synchronized.\n ",
+            "created_at": "2026-08-19T22:24:34Z"
+          },
+          {
+            "path": "src/sentry/api/paginator.py",
+            "line": 182,
+            "body": "**DOCUMENTATION**: The comment says negative offsets are safe because the queryset handles boundary conditions, but the code clamps the non-previous path precisely to avoid passing a negative start index. Please describe the actual clamping behavior instead of claiming that the queryset supports negative slicing. \n The comment contradicts the implemented guard and misrepresents how negative queryset indices behave.\n```\n# Performance optimization: clamp non-previous offsets to zero\n# to avoid invalid negative queryset slicing.\nstart_offset = max(0, offset) if not cursor.is_prev else offset\n``` ",
+            "created_at": "2026-08-19T22:24:35Z"
+          },
+          {
+            "path": "src/sentry/api/paginator.py",
+            "line": 880,
+            "body": "**PERFORMANCE_OPTIMIZATION**: This branch assigns a negative `cursor.offset` to `start_offset` and then uses it in `queryset[start_offset:stop]`. Django querysets do not support negative slice indices, so an advanced-pagination request with a negative offset will raise at runtime rather than paginate. Remove this branch or clamp the offset before slicing; the nearby claim that Django handles negative slicing should also be removed. \n With advanced features enabled, a negative cursor offset flows directly into an unsupported queryset slice and causes the request to fail.\n```\nstart_offset = max(0, offset) if not cursor.is_prev else offset\nstop = start_offset + limit + extra\nresults = list(queryset[start_offset:stop])\n``` ",
+            "created_at": "2026-08-19T22:24:35Z"
+          },
+          {
+            "path": "src/sentry/scripts/spans/add-buffer.lua",
+            "line": 47,
+            "body": "**ERROR**: These keys previously held Redis Sets, but the new script applies sorted-set commands to the same key namespace. Existing keys, or keys written by old instances during a rolling deployment, will make `zcard`/`zunionstore` fail with `WRONGTYPE`, aborting the Lua script. Add a deployment migration or use a new key namespace so set-typed and sorted-set-typed data cannot overlap. \n Redis commands cannot reinterpret an existing Set as a Sorted Set, so mixed-version or pre-deployment keys cause the buffering script to fail.\n ",
+            "created_at": "2026-08-19T22:24:36Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 8ab88145113dd23a930e23b9cbbcf8b30e4c0b17.",
+            "created_at": "2026-08-19T22:24:37Z"
+          }
+        ]
       }
     ]
   },
@@ -27930,6 +28530,43 @@
             "created_at": "2026-06-28T22:35:45Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "sentry__sentry-greptile__deputydev-v5__PR3__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/sentry__sentry-greptile__deputydev-v5__PR3__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "src/sentry/api/endpoints/organization_events_stats.py",
+            "line": 226,
+            "body": "**DOCUMENTATION**: This comment claims that aliasing `should_upsample` as `upsampling_enabled` provides query optimization and caching, but the assignment does neither. Remove the unsupported claim and either use `should_upsample` directly or assign the eligibility result directly to `upsampling_enabled`. \n The alias adds no behavior corresponding to the optimization and caching described by the comment.\n```\nupsampling_enabled = is_errors_query_for_error_upsampled_projects(\n    snuba_params, organization, dataset, request\n)\nfinal_columns = query_columns\n``` ",
+            "created_at": "2026-08-19T22:30:12Z"
+          },
+          {
+            "path": "src/sentry/api/helpers/error_upsampling.py",
+            "line": 112,
+            "body": "**ERROR**: This enables the `upsampled_count()` rewrite for the errors dataset, but the supplied dataset search finds that function registered only in Discover. An allowlisted errors-dataset request containing `count()` will therefore produce a function that its query builder cannot resolve. Register `upsampled_count` for the errors dataset or keep this branch disabled until that registration exists. \n The helper emits `upsampled_count() as count` on a dataset whose function converter does not register `upsampled_count`.\n```\nRegister `upsampled_count` in the errors dataset's function converter, or return `False` for `dataset == errors` until that registration is added.\n``` ",
+            "created_at": "2026-08-19T22:30:13Z"
+          },
+          {
+            "path": "src/sentry/api/helpers/error_upsampling.py",
+            "line": 137,
+            "body": "**SECURITY**: A raw substring check also matches queries such as `!event.type:error`, `NOT event.type:error`, quoted text containing that string, or mixed event-type expressions. Those queries are not necessarily error-focused but will still receive the upsampled aggregation. Replace this with query classification that confirms `event.type:error` is a positive filter and rejects negated, quoted, or mixed-type cases. \n False-positive classification applies the error-only column transformation to queries that exclude or include non-error events.\n ",
+            "created_at": "2026-08-19T22:30:14Z"
+          },
+          {
+            "path": "src/sentry/testutils/factories.py",
+            "line": 353,
+            "body": "**ERROR**: The truthiness check skips a numeric `client_sample_rate` of `0.0`, so the factory cannot preserve that edge case in `sample_rate`. Check explicitly for `None` before converting the value. \n Python treats numeric zero as false, preventing the assignment even though the value is present and convertible.\n```\nif client_sample_rate is not None:\n    try:\n        normalized_data[\"sample_rate\"] = float(client_sample_rate)\n    except Exception:\n        pass\n``` ",
+            "created_at": "2026-08-19T22:30:15Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 6ad6fe3899f630ccf922ce7ac681147dc7987435.",
+            "created_at": "2026-08-19T22:30:15Z"
+          }
+        ]
       }
     ]
   },
@@ -29189,6 +29826,25 @@
             "created_at": "2026-06-28T22:40:53Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "grafana__grafana__deputydev-v5__PR103633__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR103633__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "pkg/services/authz/rbac/service.go",
+            "line": 117,
+            "body": "**SECURITY**: Check the positive permission cache before returning this cached denial, or invalidate denial entries when permissions change. A denial written at line 154 currently takes unconditional precedence over both `permCache` and the database, so a later permission grant can continue returning `Allowed: false` until `shortCacheTTL` expires. This also affects action-level checks whose denial key has empty name and parent segments. \n The early return on a denial-cache hit prevents newly populated positive permissions or the database fallback from overriding a stale denial.\n```\ncachedPerms, cacheErr := s.getCachedIdentityPermissions(ctx, checkReq.Namespace, checkReq.IdentityType, checkReq.UserUID, checkReq.Action)\nif cacheErr == nil {\n\tallowed, err := s.checkPermission(ctx, cachedPerms, checkReq)\n\tif err == nil && allowed {\n\t\treturn &authzv1.CheckResponse{Allowed: true}, nil\n\t}\n}\n\nif _, ok := s.permDenialCache.Get(ctx, permDenialKey); ok {\n\treturn &authzv1.CheckResponse{Allowed: false}, nil\n}\n``` ",
+            "created_at": "2026-08-20T00:10:24Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 7562f37880367411a62304bcdcdc178bced23906.",
+            "created_at": "2026-08-20T00:10:25Z"
+          }
+        ]
       }
     ]
   },
@@ -30712,6 +31368,61 @@
             "created_at": "2026-06-28T22:48:17Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "sentry__sentry__deputydev-v5__PR67876__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/sentry__sentry__deputydev-v5__PR67876__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "src/sentry/integrations/github/integration.py",
+            "line": 434,
+            "body": "**ERROR**: Handle failures from `get_user_info` here. It raises for non-2xx GitHub responses, but this call is outside any exception handler, so an invalid token, rate limit, or GitHub failure produces an unhandled 500 instead of the intended installation error page. \n get_user_info calls raise_for_status, and its exceptions currently escape OAuthLoginView.dispatch.\n```\ntry:\n    authenticated_user_info = get_user_info(payload[\"access_token\"])\nexcept Exception:\n    return error(request, self.active_organization)\nif \"login\" not in authenticated_user_info:\n    return error(request, self.active_organization)\n``` ",
+            "created_at": "2026-08-19T22:35:40Z"
+          },
+          {
+            "path": "src/sentry/integrations/github/integration.py",
+            "line": 431,
+            "body": "**SECURITY**: Validate that the token is non-empty rather than only checking whether the key exists. A parsed response containing `access_token=` passes this condition and sends an empty token to `get_user_info`. \n The key-presence check accepts an empty credential and proceeds with an invalid authorization value.\n```\nif not payload.get(\"access_token\"):\n    return error(request, self.active_organization)\n``` ",
+            "created_at": "2026-08-19T22:35:41Z"
+          },
+          {
+            "path": "src/sentry/integrations/github/integration.py",
+            "line": 434,
+            "body": "**PERFORMANCE_OPTIMIZATION**: This adds a synchronous call to `get_user_info`, whose underlying `session.get` has no timeout. Add an explicit timeout there so an unresponsive GitHub API cannot hold the installation request indefinitely. \n The user-info request has no bounded wait and can retain a request worker during GitHub API degradation.\n```\nresp = session.get(\n    \"https://api.github.com/user\",\n    headers={\n        \"Accept\": \"application/vnd.github.machine-man-preview+json\",\n        \"Authorization\": f\"token {access_token}\",\n    },\n    timeout=10,\n)\n``` ",
+            "created_at": "2026-08-19T22:35:42Z"
+          },
+          {
+            "path": "src/sentry/integrations/github/integration.py",
+            "line": 494,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Reuse the `Integration` loaded for the organization-membership check instead of querying the same installation again here. Retaining that object and checking its `status` removes one database round trip from this path. \n The same integration row is fetched at +478 and +494 whenever it exists without an OrganizationIntegration.\n```\ntry:\n    integration = Integration.objects.get(external_id=installation_id)\n    installations_exist = OrganizationIntegration.objects.filter(\n        integration=integration\n    ).exists()\nexcept Integration.DoesNotExist:\n    return pipeline.next_step()\n\nif integration.status != ObjectStatus.ACTIVE:\n    return error(request, self.active_organization)\n``` ",
+            "created_at": "2026-08-19T22:35:43Z"
+          },
+          {
+            "path": "src/sentry/integrations/github/integration.py",
+            "line": 423,
+            "body": "**CODE_MAINTAINABILTIY**: Move `safe_urlopen` inside the following `try` block. Network failures raised while opening the token endpoint currently escape as a 500; only failures while reading or parsing the response are converted into the controlled installation error. \n The exception handler begins after the network request, leaving connection, TLS, and timeout failures unhandled.\n```\ntry:\n    req = safe_urlopen(url=ghip.get_oauth_access_token_url(), data=data)\n    body = safe_urlread(req).decode(\"utf-8\")\n    payload = dict(parse_qsl(body))\nexcept Exception:\n    payload = {}\n``` ",
+            "created_at": "2026-08-19T22:35:44Z"
+          },
+          {
+            "path": "src/sentry/integrations/github/integration.py",
+            "line": 482,
+            "body": "**ERROR**: This early return advances a new installation without verifying that `github_authenticated_user` matches the GitHub user who installed the app. The only sender comparison is below this branch and requires an existing `Integration`, so a supplied installation ID with no record bypasses the identity check entirely. Do not advance this path until the installer identity can be validated. \n Integration.DoesNotExist returns before the sole authenticated-user versus sender comparison.\n ",
+            "created_at": "2026-08-19T22:35:45Z"
+          },
+          {
+            "path": "src/sentry/integrations/github/integration.py",
+            "line": 503,
+            "body": "**ERROR**: Guard the nested metadata lookup before comparing users. `sender` is only conditionally populated, so an existing integration without `sender` or `login` raises `KeyError` here instead of returning the controlled validation error. \n The direct nested dictionary access assumes metadata fields that the supplied evidence shows are optional.\n```\nsender_login = (integration.metadata.get(\"sender\") or {}).get(\"login\")\nif (\n    not sender_login\n    or pipeline.fetch_state(\"github_authenticated_user\") != sender_login\n):\n    return error(request, self.active_organization)\n``` ",
+            "created_at": "2026-08-19T22:35:46Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit bb75657fc8f13923c1d7983f422290908a1e7310.",
+            "created_at": "2026-08-19T22:35:47Z"
+          }
+        ]
       }
     ]
   },
@@ -31966,6 +32677,25 @@
             "created_at": "2026-06-28T22:22:01Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "keycloak__keycloak__deputydev-v5__PR32918__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR32918__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/organization/cache/OrganizationCacheTest.java",
+            "line": 381,
+            "body": "**ERROR**: Register cleanup with the actual alias created by the test. Both this line and line 425 use the literal `\"alias\"`, while the created providers are named `\"idp-alias-\" + i` and `\"idp-alias-20\"`. Consequently, cleanup targets a non-existent provider and leaves the created IDPs in the test realm, potentially contaminating subsequent tests. \n The cleanup callbacks do not target any of the IDPs created by this test, so those providers remain after the test completes.\n```\n// In the loop:\ngetCleanup().addCleanup(testRealm().identityProviders().get(\"idp-alias-\" + i)::remove);\n\n// For idp-alias-20:\ngetCleanup().addCleanup(testRealm().identityProviders().get(\"idp-alias-20\")::remove);\n``` ",
+            "created_at": "2026-08-19T20:33:55Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit e944e2acf1e64421860a8e827f93ad06c169621b.",
+            "created_at": "2026-08-19T20:33:56Z"
+          }
+        ]
       }
     ]
   },
@@ -33117,6 +33847,31 @@
             "created_at": "2026-06-28T23:57:04Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "grafana__grafana__deputydev-v5__PR94942__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR94942__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "pkg/expr/reader.go",
+            "line": 194,
+            "body": "**CODE_MAINTAINABILTIY**: `enableSqlExpressions` performs a feature-flag lookup but returns `false` on both paths, so the lookup is dead and the flag-shaped logic misleadingly suggests that this security kill switch can be enabled. Express the permanent disablement directly and document why it must remain disabled. \n The current structure discards the flag result, incurs an unnecessary lookup, and obscures that returning false is an intentional security measure, increasing the risk of a future maintainer accidentally re-enabling the path.\n```\n// enableSqlExpressions is intentionally hardcoded to false to keep SQL expressions disabled.\nfunc enableSqlExpressions(_ *ExpressionQueryReader) bool {\n\treturn false\n}\n``` ",
+            "created_at": "2026-08-20T00:49:56Z"
+          },
+          {
+            "path": "pkg/expr/sql/db.go",
+            "line": 1,
+            "body": "**DOCUMENTATION**: Please document that this package intentionally provides a disabled stub whose operations always fail. Without that context, the exported `DB` API and its `not implemented` methods look like unfinished scaffolding rather than a deliberate security boundary. \n The new exported API contains only failing stubs, but the file provides no explanation that this behavior is intentional, making later accidental implementation more likely.\n```\n// Package sql provides a disabled stub implementation of the in-memory SQL database.\n// Its operations intentionally return errors to prevent SQL execution.\npackage sql\n``` ",
+            "created_at": "2026-08-20T00:49:57Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit f3317b329b4eb8fd96f99dd86525bc4a22d20248.",
+            "created_at": "2026-08-20T00:49:58Z"
+          }
+        ]
       }
     ]
   },
@@ -34108,6 +34863,25 @@
             "created_at": "2026-06-29T00:05:52Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "grafana__grafana__deputydev-v5__PR90939__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR90939__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "pkg/api/webassets/webassets.go",
+            "line": 48,
+            "body": "**ERROR**: Re-check `entryPointAssetsCache` after acquiring the write lock. Multiple non-Dev callers can observe `ret == nil` before any of them obtains this lock; after the first caller populates the cache, each waiter still proceeds to repeat the filesystem or CDN load and overwrite the cached result. Return the populated cache under the write lock before starting that work. \n The read lock is released before the write lock is acquired, so all callers that observed an empty cache can serially execute the expensive initialization path even after the first caller has populated it.\n```\nentryPointAssetsCacheMu.Lock()\ndefer entryPointAssetsCacheMu.Unlock()\n\nif cfg.Env != setting.Dev && entryPointAssetsCache != nil {\n    return entryPointAssetsCache, nil\n}\n\nvar err error\nvar result *dtos.EntryPointAssets\n``` ",
+            "created_at": "2026-08-20T00:43:56Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit b1613e320acff00623e6efc59f00ee68c7684a97.",
+            "created_at": "2026-08-20T00:43:57Z"
+          }
+        ]
       }
     ]
   },
@@ -35513,6 +36287,43 @@
             "created_at": "2026-06-29T00:00:06Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "grafana__grafana__deputydev-v5__PR80329__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR80329__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "pkg/services/annotations/annotationsimpl/cleanup_test.go",
+            "line": 116,
+            "body": "**CODE_MAINTAINABILTIY**: Register `t.Cleanup` before creating or asserting the test data. These subtests share `fakeSQL`, so if `createTestAnnotations` or either initial assertion calls `t.Fatal`, teardown has not yet been registered and the remaining rows can pollute subsequent subtests. \n A fatal setup failure exits the subtest before cleanup registration, leaving data in the shared test database.\n```\nt.Run(test.name, func(t *testing.T) {\n\tt.Cleanup(func() {\n\t\terr := fakeSQL.WithDbSession(context.Background(), func(session *db.Session) error {\n\t\t\t_, deleteAnnotationErr := session.Exec(\"DELETE FROM annotation\")\n\t\t\t_, deleteAnnotationTagErr := session.Exec(\"DELETE FROM annotation_tag\")\n\t\t\treturn errors.Join(deleteAnnotationErr, deleteAnnotationTagErr)\n\t\t})\n\t\tassert.NoError(t, err)\n\t})\n\n\tcreateTestAnnotations(t, fakeSQL, test.createAnnotationsNum, test.createOldAnnotationsNum)\n\tassertAnnotationCount(t, fakeSQL, \"\", int64(test.createAnnotationsNum))\n\tassertAnnotationTagCount(t, fakeSQL, 2*int64(test.createAnnotationsNum))\n})\n``` ",
+            "created_at": "2026-08-20T00:38:35Z"
+          },
+          {
+            "path": "pkg/services/annotations/annotationsimpl/xorm_store.go",
+            "line": 609,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Avoid rebuilding the complete `values` string with `fmt.Sprintf` on every iteration. Each iteration copies the growing prefix, making this path quadratic for large SQLite batches such as the new 32,767-ID case. Build the list incrementally with `strings.Builder` and `strconv.FormatInt`. \n Repeatedly formatting the growing ID list causes O(n²) copying and allocations on the large-batch SQLite path.\n```\nvar sb strings.Builder\nfor i, v := range ids {\n\tif i > 0 {\n\t\tsb.WriteString(\", \")\n\t}\n\tsb.WriteString(strconv.FormatInt(v, 10))\n}\nsql = fmt.Sprintf(`DELETE FROM %s WHERE id IN (%s)`, table, sb.String())\n``` ",
+            "created_at": "2026-08-20T00:38:36Z"
+          },
+          {
+            "path": "pkg/services/annotations/annotationsimpl/xorm_store.go",
+            "line": 534,
+            "body": "**PERFORMANCE_OPTIMIZATION**: These are routine cleanup diagnostics, but all six calls use `Error`, and the pre-delete calls also emit the complete `ids` slice and raw `cond`. A configured batch can contain tens of thousands of IDs, producing very large false error events that expose row IDs and query details on every batch. Log only summary counts at `Debug`, and reserve `Error` for an actual returned error. \n Successful cleanup batches generate oversized error-level entries containing full ID lists and SQL conditions, creating log noise and exposing operational details.\n```\nr.log.Debug(\"Annotations to clean by time\", \"count\", len(ids))\n\nx, err := r.deleteByIDs(ctx, \"annotation\", ids)\nif err != nil {\n\tr.log.Error(\"Failed to clean annotations by time\", \"count\", len(ids), \"affected\", x, \"err\", err)\n} else {\n\tr.log.Debug(\"Cleaned annotations by time\", \"count\", len(ids), \"affected\", x)\n}\nreturn x, err\n``` ",
+            "created_at": "2026-08-20T00:38:37Z"
+          },
+          {
+            "path": "pkg/services/cleanup/cleanup.go",
+            "line": 77,
+            "body": "**CODE_MAINTAINABILTIY**: This changes the shared cleanup schedule from every 10 minutes to every minute, so every cleanup job—not only annotation cleanup—is now eligible to run ten times as often. That increases database work across snapshots, dashboard versions, images, annotations, and the other jobs in the cycle. Restore the previous interval unless the higher frequency is intentional, or isolate the cleanup that requires it. \n The shared ticker change increases the execution frequency and database load of all cleanup jobs by up to tenfold.\n```\nticker := time.NewTicker(time.Minute * 10)\n``` ",
+            "created_at": "2026-08-20T00:38:38Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 04cfa3bfd469c035ff8b35f9a66867c4e8d5dcf4.",
+            "created_at": "2026-08-20T00:38:39Z"
+          }
+        ]
       }
     ]
   },
@@ -37355,6 +38166,79 @@
             "created_at": "2026-06-28T23:53:28Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "grafana__grafana__deputydev-v5__PR90045__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR90045__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+            "line": 51,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Detach the async legacy writes from the request cancellation chain before applying the 10-second timeout. As written, `context.WithTimeoutCause(ctx, ...)` inherits cancellation from the request, so a completed or cancelled request can abort Create, Delete, Update, and DeleteCollection legacy writes before their timeout and leave legacy storage stale. \n All four fire-and-forget operations use the request context as their timeout context's parent, allowing request cancellation to terminate synchronization early.\n```\ndetachedCtx := context.WithoutCancel(ctx)\nctx, cancel := context.WithTimeoutCause(detachedCtx, time.Second*10, errors.New(\"legacy create timeout\"))\ndefer cancel()\n``` ",
+            "created_at": "2026-08-20T00:33:18Z"
+          },
+          {
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+            "line": 166,
+            "body": "**CODE_MAINTAINABILTIY**: Record this asynchronous `d.Legacy.DeleteCollection` call with `recordLegacyDuration`, not `recordStorageDuration`. The current call puts legacy latency and failures into the storage histogram and leaves the legacy DeleteCollection series incomplete. \n The measured operation is performed by Legacy storage and uses `startLegacy`, but its observation is routed to the storage histogram.\n```\nd.recordLegacyDuration(err != nil, mode3Str, options.Kind, method, startLegacy)\n``` ",
+            "created_at": "2026-08-20T00:33:20Z"
+          },
+          {
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+            "line": 94,
+            "body": "**DOCUMENTATION**: Please add a leading comment for `Delete` describing that it deletes from Storage synchronously and schedules the LegacyStorage deletion asynchronously, matching the documentation pattern used by the other `DualWriterMode3` methods. \n Delete is the only method in this implementation without a comment describing its storage behavior.\n```\n// Delete deletes from Storage synchronously and then asynchronously deletes from LegacyStorage.\nfunc (d *DualWriterMode3) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {\n``` ",
+            "created_at": "2026-08-20T00:33:21Z"
+          },
+          {
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+            "line": 55,
+            "body": "**DOCUMENTATION**: Log failures from each asynchronous legacy write before recording its metric. Create, Delete, Update, and DeleteCollection currently discard the returned error after metric recording, so legacy divergence has no corresponding error entry in production logs. \n Every async legacy call captures an error but only converts it to a metric label, leaving failures absent from logs.\n```\nif errObjectSt != nil {\n\tlog.Error(errObjectSt, \"async legacy create failed\")\n}\nd.recordLegacyDuration(errObjectSt != nil, mode3Str, options.Kind, method, startLegacy)\n``` ",
+            "created_at": "2026-08-20T00:33:21Z"
+          },
+          {
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+            "line": 129,
+            "body": "**CODE_MAINTAINABILTIY**: Use `recordStorageDuration` on this error path. The failing call is `d.Storage.Update` and the timer is `startStorage`; recording it through `recordLegacyDuration` misattributes storage failures and latency to the legacy histogram. \n A Storage.Update failure is currently written to the separate legacy metric series.\n```\nd.recordStorageDuration(true, mode3Str, options.Kind, method, startStorage)\n``` ",
+            "created_at": "2026-08-20T00:33:22Z"
+          },
+          {
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+            "line": 50,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Bound concurrency for the asynchronous legacy writes. Each successful write request currently starts another goroutine immediately, so a burst can create many concurrent legacy calls and retain their request objects and callbacks until those calls finish or time out. Use a semaphore or bounded worker queue to apply backpressure. \n Create, Delete, Update, and DeleteCollection launch goroutines without any shared concurrency limit.\n```\nlegacySem chan struct{}\n\n// Before the legacy call:\nd.legacySem <- struct{}{}\ndefer func() { <-d.legacySem }()\n``` ",
+            "created_at": "2026-08-20T00:33:23Z"
+          },
+          {
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+            "line": 97,
+            "body": "**ERROR**: Place the enriched `log` variable into the context rather than the base `d.Log`. The current call drops the `name`, `kind`, and `method` fields from any downstream logging that retrieves its logger from `ctx`. \n The logger enriched on the preceding line is discarded when constructing the Delete context.\n```\nctx = klog.NewContext(ctx, log)\n``` ",
+            "created_at": "2026-08-20T00:33:24Z"
+          },
+          {
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+            "line": 45,
+            "body": "**SECURITY**: Use `recordStorageDuration` on this error path. The failing call is `d.Storage.Create` and the timer is `startStorage`; recording it through `recordLegacyDuration` creates a legacy observation even though no legacy operation ran. \n A Storage.Create failure is currently written to the separate legacy metric series.\n```\nd.recordStorageDuration(true, mode3Str, options.Kind, method, startStorage)\n``` ",
+            "created_at": "2026-08-20T00:33:25Z"
+          },
+          {
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+            "line": 106,
+            "body": "**ERROR**: Pass `options.Kind` as the third metric argument, as the error path does. This argument maps to the `kind` label, so using the per-object `name` creates a separate histogram series for each deleted object and prevents kind-level aggregation. \n The Delete success path supplies an object identifier to a label position defined for resource kind.\n```\nd.recordStorageDuration(false, mode3Str, options.Kind, method, startStorage)\n``` ",
+            "created_at": "2026-08-20T00:33:25Z"
+          },
+          {
+            "path": "pkg/apiserver/rest/dualwriter_mode3_test.go",
+            "line": 63,
+            "body": "**CODE_MAINTAINABILTIY**: Synchronize these tests with the asynchronous legacy operation and assert the mock expectations before the test returns. Currently the Create, Delete, Update, and DeleteCollection tests only validate the synchronous Storage result; a legacy goroutine that never runs still leaves the test green. \n Configured legacy expectations are neither awaited nor asserted, so the mode-3 legacy-write behavior is unverified.\n```\n// Signal completion from the legacy mock, wait for that signal after the operation,\n// then verify the configured calls.\nm.AssertExpectations(t)\n``` ",
+            "created_at": "2026-08-20T00:33:26Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit e369f24665ec70e1e3700f457d25ef83301d931a.",
+            "created_at": "2026-08-20T00:33:27Z"
+          }
+        ]
       }
     ]
   },
@@ -38856,6 +39740,49 @@
             "created_at": "2026-06-28T23:38:08Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "grafana__grafana__deputydev-v5__PR106778__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR106778__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "public/app/features/alerting/unified/components/rule-viewer/AlertRuleMenu.tsx",
+            "line": 87,
+            "body": "**ERROR**: The Prometheus ability path can make `canPause` or `canDelete` true when only `promRule` is available, but the existing menu guards still require a `rulerRule`: Pause checks `rulerRuleType.grafana.rule(rulerRule)`, and Delete checks `canDelete && rulerRule`. Consequently, neither action is exposed in the new prom-only list path. Add prom-rule-compatible action paths, or avoid treating these actions as available without the ruler data their handlers require. \n Prometheus-derived permissions are calculated, but ruler-only rendering guards prevent Pause and Delete from appearing for prom-only rules.\n ",
+            "created_at": "2026-08-20T00:27:25Z"
+          },
+          {
+            "path": "public/app/features/alerting/unified/hooks/useAbilities.ts",
+            "line": 289,
+            "body": "**DOCUMENTATION**: Please remove or clarify the `// duplicate` comment. It does not identify what is duplicated or explain any intended follow-up, so it obscures rather than documents this permission check. \n The unexplained marker does not communicate intent to future maintainers.\n```\nconst { isEditable, isRemovable, loading } = useIsGrafanaPromRuleEditable(rule);\n``` ",
+            "created_at": "2026-08-20T00:27:26Z"
+          },
+          {
+            "path": "public/app/features/alerting/unified/hooks/useAbilities.ts",
+            "line": 293,
+            "body": "**PERFORMANCE_OPTIMIZATION**: `useAllGrafanaPromRuleAbilities` subscribes to the same folder twice: `useIsGrafanaPromRuleEditable(rule)` calls `useFolder(folderUid)`, while `useCanSilenceInFolder(folderUid)` reaches another `useFolder(folderUid)` through `useFolderPermissions`. Consolidate the folder lookup and derive both edit/delete and silence permissions from that result to avoid duplicate hook subscriptions for every rule row. \n Each ability evaluation registers two folder subscriptions for the same UID, adding redundant per-row work in the rule list.\n ",
+            "created_at": "2026-08-20T00:27:27Z"
+          },
+          {
+            "path": "public/app/features/alerting/unified/rule-list/components/RuleActionsButtons.V2.tsx",
+            "line": 146,
+            "body": "**DOCUMENTATION**: This warning is identical for every failure and does not identify the input that could not be converted. Include available context from `groupIdentifier`, `rule`, or `promRule` so a production occurrence can be associated with the failing rule path. \n The static warning cannot distinguish which rule or group caused identifier construction to fail.\n ",
+            "created_at": "2026-08-20T00:27:28Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "**SECURITY**: In the new list path, `RuleActionsButtons` receives only `promRule`, so `rulerRuleType.grafana.alertingRule(rule)` is false. The menu can still grant Silence through the Prometheus ability and set `showSilenceDrawer`, but this guard prevents the drawer from mounting, making the action a no-op. Provide a prom-rule-compatible silence path; the current drawer also requires a ruler DTO and therefore cannot simply receive the absent `rule`. \n The prom-only Silence action is visible and clickable, but its ruler-only drawer condition prevents any UI from opening.\n ",
+            "created_at": "2026-08-20T00:27:29Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 8df850371034b73dc4dd9908cc30fa09f12a1f97.",
+            "created_at": "2026-08-20T00:27:30Z"
+          }
+        ]
       }
     ]
   },
@@ -39992,6 +40919,25 @@
             "created_at": "2026-06-28T23:15:44Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "grafana__grafana__deputydev-v5__PR107534__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR107534__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": null,
+            "line": null,
+            "body": "LGTM!!",
+            "created_at": "2026-08-20T00:20:16Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 1cab4143d443c33079566f6ca36931eb8468159f.",
+            "created_at": "2026-08-20T00:20:16Z"
+          }
+        ]
       }
     ]
   },
@@ -41466,6 +42412,37 @@
             "created_at": "2026-06-28T22:20:35Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "grafana__grafana__deputydev-v5__PR79265__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR79265__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "pkg/services/anonymous/anonimpl/anonstore/database.go",
+            "line": 96,
+            "body": "**ERROR**: `RowsAffected() == 0` does not prove that this is a new device blocked by the limit. The UPDATE also filters on `updated_at BETWEEN ...`, so an existing device outside that window reaches this branch and is incorrectly denied with `ErrDeviceLimitReached`. Distinguish an unknown device from a timestamp mismatch, or update any row matching the existing `device_id`, and return the limit error only when the device is genuinely absent while the limit is full. \n An existing device whose stored timestamp falls outside the UPDATE window affects zero rows and is misclassified as exceeding the device limit, causing authentication denial.\n```\nOnly return ErrDeviceLimitReached after determining that no row exists for device.DeviceID while the configured limit is full. Alternatively, remove the updated_at range from the UPDATE predicate so any existing device_id can be updated.\n``` ",
+            "created_at": "2026-08-20T00:03:30Z"
+          },
+          {
+            "path": "pkg/services/anonymous/anonimpl/api/api.go",
+            "line": 18,
+            "body": "**CODE_MAINTAINABILTIY**: `anonymousDeviceExpiration` is now independently defined as 30 days in both this package and `anonstore`. These values govern device listing and limit enforcement respectively, so changing only one would silently make those windows inconsistent. Define the expiration once and reuse it in both packages. \n Independent constants for the same expiration policy can diverge and produce inconsistent listing and device-limit windows.\n```\nExport the store constant as `anonstore.AnonymousDeviceExpiration`, remove this local constant, and use `time.Now().Add(-anonstore.AnonymousDeviceExpiration)` here.\n``` ",
+            "created_at": "2026-08-20T00:03:31Z"
+          },
+          {
+            "path": "pkg/services/anonymous/anonimpl/impl.go",
+            "line": 147,
+            "body": "**ERROR**: `tagDeviceUI` populates the local cache before calling `CreateOrUpdateDevice`. When that call returns `ErrDeviceLimitReached`, this path now propagates the error and blocks the current request, but the cache entry remains. The next request for the same device then returns successfully from the cache without checking the store, bypassing the limit. Only cache the device after `CreateOrUpdateDevice` succeeds. \n A failed device registration leaves a successful cache marker, allowing subsequent authentication attempts to skip device-limit enforcement.\n```\nif err := a.anonStore.CreateOrUpdateDevice(ctx, device); err != nil {\n\treturn err\n}\na.localCache.SetDefault(key, struct{}{})\nreturn nil\n``` ",
+            "created_at": "2026-08-20T00:03:32Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit bbd8c507cdf56f6b884373809e0b24abd4a2353d.",
+            "created_at": "2026-08-20T00:03:33Z"
+          }
+        ]
       }
     ]
   },
@@ -42570,6 +43547,25 @@
             "created_at": "2026-06-28T23:57:17Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "discourse__discourse-graphite__deputydev-v5__PR9__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR9__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "config/initializers/i18n.rb",
+            "line": 12,
+            "body": "**CODE_MAINTAINABILTIY**: Avoid subclassing `Hash` here. `FallbackLocaleList#[]` ignores Hash's stored entries and always computes a fallback list, making inherited storage operations inconsistent with lookup. Use a plain class that exposes only the required `[]` and `ensure_loaded!` methods. \n Values stored through inherited Hash operations cannot be retrieved through the overridden `[]`, so the class does not preserve Hash lookup semantics.\n```\nclass FallbackLocaleList\n  def [](locale)\n    # user locale, site locale, english\n    # TODO - this can be extended to be per-language for a better user experience\n    # (e.g. fallback zh_TW to zh_CN / vice versa)\n    [locale, SiteSetting.default_locale.to_sym, :en].uniq.compact\n  end\n\n  def ensure_loaded!\n    self[I18n.locale].each { |l| I18n.ensure_loaded! l }\n  end\nend\n``` ",
+            "created_at": "2026-08-19T22:07:29Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293.",
+            "created_at": "2026-08-19T22:07:30Z"
+          }
+        ]
       }
     ]
   },
@@ -43733,6 +44729,37 @@
             "created_at": "2026-06-28T22:55:56Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "grafana__grafana__deputydev-v5__PR76186__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR76186__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "pkg/plugins/log/fake.go",
+            "line": 47,
+            "body": "**ERROR**: `FromContext` returns a fresh `TestLogger`, so calls such as `m.logger.FromContext(ctx).Info(...)` update a throwaway instance rather than the logger held by the test. Return `f` here so assertions on the original logger's `InfoLogs`, `WarnLogs`, and other stores can observe calls made through `FromContext`. \n `NewTestLogger()` owns independent zeroed log stores, disconnecting context-derived log calls from the injected test logger.\n```\nfunc (f *TestLogger) FromContext(_ context.Context) Logger {\n\treturn f\n}\n``` ",
+            "created_at": "2026-08-20T00:16:04Z"
+          },
+          {
+            "path": "pkg/services/pluginsintegration/clientmiddleware/contextual_logger_middleware.go",
+            "line": 1,
+            "body": "**CODE_MAINTAINABILTIY**: Please add unit tests for this new middleware. In particular, cover the endpoint attributes added by each instrumented method, optional user and datasource fields, and delegation with nil optional `PluginContext` fields. \n No corresponding test file exercises the newly extracted context-enrichment and middleware delegation behavior.\n ",
+            "created_at": "2026-08-20T00:16:05Z"
+          },
+          {
+            "path": "pkg/services/pluginsintegration/clientmiddleware/logger_middleware.go",
+            "line": 36,
+            "body": "**DOCUMENTATION**: Document that `logRequest` expects `ctx` to have been enriched by `ContextualLoggerMiddleware`. Without that ordering, the emitted entry lacks the plugin, endpoint, datasource, and user attributes that were previously added directly here. \n The refactor replaces explicit field construction with attributes retrieved through `FromContext`, creating an implicit middleware-order dependency.\n```\n// logRequest executes fn and logs the result. It relies on the context having\n// been enriched with plugin and request attributes by ContextualLoggerMiddleware\n// earlier in the middleware chain.\nfunc (m *LoggerMiddleware) logRequest(ctx context.Context, fn func(ctx context.Context) error) error {\n``` ",
+            "created_at": "2026-08-20T00:16:06Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 303cdc2caf256aa5df660611120c2e3f7392365d.",
+            "created_at": "2026-08-20T00:16:07Z"
+          }
+        ]
       }
     ]
   },
@@ -45852,6 +46879,73 @@
             "created_at": "2026-06-28T23:54:24Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "discourse__discourse-graphite__deputydev-v5__PR10__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR10__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "app/controllers/admin/embeddable_hosts_controller.rb",
+            "line": 10,
+            "body": "**ERROR**: Handle a missing host before calling `save_host` or `destroy`. Both lookups use `.first`, so an unknown or already-deleted ID produces `nil`; update then dereferences it in `save_host`, while destroy calls `nil.destroy`, returning a 500 instead of a not-found response. \n A user-supplied ID that does not identify an `EmbeddableHost` causes an immediate `NoMethodError` in both actions.\n```\ndef update\n  host = EmbeddableHost.where(id: params[:id]).first\n  raise Discourse::NotFound unless host\n  save_host(host)\nend\n\ndef destroy\n  host = EmbeddableHost.where(id: params[:id]).first\n  raise Discourse::NotFound unless host\n  host.destroy\n  render json: success_json\nend\n``` ",
+            "created_at": "2026-08-19T22:13:00Z"
+          },
+          {
+            "path": "app/models/embeddable_host.rb",
+            "line": 17,
+            "body": "**SECURITY**: Normalize both sides of this case-insensitive lookup. The database value is lowercased, but the `uri.host` parameter is not, so a mixed-case URL host can fail to match an otherwise allowed host. \n `lower(host)` cannot equal a mixed-case query parameter under a case-sensitive comparison, causing valid hosts to be rejected.\n```\nwhere(\"lower(host) = lower(?)\", host).first\n``` ",
+            "created_at": "2026-08-19T22:13:01Z"
+          },
+          {
+            "path": "app/models/embeddable_host.rb",
+            "line": 6,
+            "body": "**ERROR**: Guard `host` before normalizing it. If the attribute is absent, this callback calls `sub!` on `nil` and raises before the format validation can return a normal validation error. \n Saving an `EmbeddableHost` without a host value raises `NoMethodError` during `before_validation`.\n```\nbefore_validation do\n  if self.host.present?\n    self.host.sub!(/^https?:\\/\\//, '')\n    self.host.sub!(/\\/.*$/, '')\n  end\nend\n``` ",
+            "created_at": "2026-08-19T22:13:02Z"
+          },
+          {
+            "path": "db/migrate/20150818190757_create_embeddable_hosts.rb",
+            "line": 25,
+            "body": "**SECURITY**: Quote each host value instead of interpolating it directly into this SQL literal. A stored host containing a single quote will break the migration query and can alter the SQL being executed. \n Values read from the free-text site setting are inserted into raw SQL without database quoting or bind parameters.\n```\nrecords.each do |h|\n  quoted_host = ActiveRecord::Base.connection.quote(h)\n  execute \"INSERT INTO embeddable_hosts (host, category_id, created_at, updated_at) VALUES (#{quoted_host}, #{category_id}, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)\"\nend\n``` ",
+            "created_at": "2026-08-19T22:13:05Z"
+          },
+          {
+            "path": "db/migrate/20150818190757_create_embeddable_hosts.rb",
+            "line": 9,
+            "body": "**ERROR**: Check whether this query returned a row before accessing `[0]['id']`. If the `embed_category` setting is absent or its value matches no category, `[0]` is nil and the migration aborts before reaching the uncategorized-category fallback. \n An empty category query result causes a `NoMethodError` during migration.\n```\nresult = execute(\"SELECT c.id FROM categories AS c\n                    INNER JOIN site_settings AS s ON s.value = c.name\n                    WHERE s.name = 'embed_category'\")\ncategory_id = (result && result.cmd_tuples > 0) ? result[0]['id'].to_i : 0\n``` ",
+            "created_at": "2026-08-19T22:13:06Z"
+          },
+          {
+            "path": "db/migrate/20150818190757_create_embeddable_hosts.rb",
+            "line": 3,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Add an index matching the `lower(host)` lookup used by `record_for_host`. This table currently has no index on that expression, so each host authorization lookup requires scanning the table. \n The recurring `WHERE lower(host) = ?` query has no matching functional index.\n ",
+            "created_at": "2026-08-19T22:13:07Z"
+          },
+          {
+            "path": "spec/controllers/admin/embeddable_hosts_controller_spec.rb",
+            "line": 5,
+            "body": "**CODE_MAINTAINABILTIY**: Add action-level coverage for this controller. The only test asserts inheritance, leaving create, update, destroy, validation failures, authorization, and missing-record responses untested. \n The newly added CRUD behavior and its error paths are not exercised by this spec.\n ",
+            "created_at": "2026-08-19T22:13:08Z"
+          },
+          {
+            "path": "spec/fabricators/category_fabricator.rb",
+            "line": 1,
+            "body": "**ERROR**: Restore the category fabricators to this file and place `Fabricator(:embeddable_host)` in `embeddable_host_fabricator.rb`. The definitions have been assigned to files named for the opposite model, making fabricators difficult to locate and breaking the existing file organization. \n `category_fabricator.rb` now defines only an embeddable host after removing all of its category definitions.\n ",
+            "created_at": "2026-08-19T22:13:09Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "**CODE_MAINTAINABILTIY**: Remove the corresponding `embeddable_hosts` and `embed_category` entries from the non-English server locale files as well. These setting keys are deleted here, but the supplied locale search shows their translations remain in files such as `server.es.yml` and `server.fi.yml`. \n Translations for settings removed by this change remain as orphaned locale entries.\n ",
+            "created_at": "2026-08-19T22:13:04Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit d1c69189f3c90ecf56013a8da904da9bff9a8e19.",
+            "created_at": "2026-08-19T22:13:10Z"
+          }
+        ]
       }
     ]
   },
@@ -47379,6 +48473,49 @@
             "created_at": "2026-06-28T23:37:06Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "discourse__discourse-graphite__deputydev-v5__PR7__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR7__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "app/assets/stylesheets/common/base/discourse.scss",
+            "line": 73,
+            "body": "**DOCUMENTATION**: Update this comment to describe the new theme-aware default. It still says the value is `scale-color($primary, $lightness: 50%)`, but the related coldmap rule now uses `dark-light-choose(...)`. \n The comment documents the old expression rather than the theme-aware value introduced by this change.\n```\n// the default for table cells in topic list\n// is dark-light-choose(scale-color($primary, $lightness: 50%), scale-color($secondary, $lightness: 50%))\n// numbers get dimmer as they get colder\n``` ",
+            "created_at": "2026-08-19T21:57:40Z"
+          },
+          {
+            "path": "app/assets/stylesheets/desktop/topic-post.scss",
+            "line": 291,
+            "body": "**CODE_MAINTAINABILTIY**: Preserve the original `30%` lightness in the first branch. `dark-light-choose` uses this branch on light themes, so changing it to `70%` makes `.topic-meta-data h5 a` substantially lighter than before instead of limiting the change to dark-theme rendering. \n The migration changes the existing light-theme output from 30% to 70% lightness and reduces the link's previous contrast.\n```\ncolor: dark-light-choose(scale-color($primary, $lightness: 30%), scale-color($secondary, $lightness: 70%));\n``` ",
+            "created_at": "2026-08-19T21:57:41Z"
+          },
+          {
+            "path": "app/assets/stylesheets/desktop/user.scss",
+            "line": 522,
+            "body": "**CODE_MAINTAINABILTIY**: Preserve the original `30%` light-theme value for `.group-member-info .name` here and in `mobile/user.scss`. Both replacements use `50%`, making `.name` identical in lightness to the adjacent `.title` and removing their previous visual distinction. \n Desktop and mobile both change member names from 30% to 50% lightness on light themes, collapsing the existing name/title hierarchy.\n```\ncolor: dark-light-choose(scale-color($primary, $lightness: 30%), scale-color($secondary, $lightness: 70%));\n``` ",
+            "created_at": "2026-08-19T21:57:42Z"
+          },
+          {
+            "path": "app/assets/stylesheets/mobile/modal.scss",
+            "line": 102,
+            "body": "**SECURITY**: Keep `70%` in the first branch. This branch controls light themes, so replacing the original `70%` with `30%` makes `.custom-message-length` much darker and more prominent than before; the desktop counterpart also preserves `70%` for light themes. \n The mobile conversion reverses the original light-theme lightness and diverges from the equivalent desktop rule.\n```\ncolor: dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%));\n``` ",
+            "created_at": "2026-08-19T21:57:43Z"
+          },
+          {
+            "path": "app/assets/stylesheets/mobile/topic-post.scss",
+            "line": 182,
+            "body": "**CODE_MAINTAINABILTIY**: Preserve the original `20%` lightness in the first branch. Using `50%` makes this `h3` substantially lighter on light themes and removes its previous emphasis; the equivalent desktop rule correctly keeps `20%`. \n The mobile migration changes the heading's light-theme output from 20% to 50% and diverges from the desktop equivalent.\n```\ncolor: dark-light-choose(scale-color($primary, $lightness: 20%), scale-color($secondary, $lightness: 80%));\n``` ",
+            "created_at": "2026-08-19T21:57:44Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit d38c4d5f7443223c81529c22470293771baf9f38.",
+            "created_at": "2026-08-19T21:57:45Z"
+          }
+        ]
       }
     ]
   },
@@ -49374,6 +50511,55 @@
             "created_at": "2026-06-28T23:44:35Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "discourse__discourse-graphite__deputydev-v5__PR8__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR8__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "app/assets/javascripts/admin/controllers/admin-group.js.es6",
+            "line": 13,
+            "body": "**CODE_MAINTAINABILTIY**: `Math.floor(user_count / limit) + 1` creates a phantom page whenever the member count is an exact multiple of the page size. For example, 50 members at a limit of 50 produces two pages, allowing navigation to an empty result. Use `Math.ceil(user_count / limit)` instead. \n The formula always adds one after division, so exact multiples overcount the number of pages and permit an unnecessary empty-page request.\n```\ntotalPages: function() {\n  if (this.get(\"user_count\") == 0) { return 0; }\n  return Math.ceil(this.get(\"user_count\") / this.get(\"limit\"));\n}.property(\"limit\", \"user_count\"),\n``` ",
+            "created_at": "2026-08-19T22:03:06Z"
+          },
+          {
+            "path": "app/assets/javascripts/admin/controllers/admin-group.js.es6",
+            "line": 56,
+            "body": "**CODE_MAINTAINABILTIY**: Handle the promise returned by `removeMember`. As written, a server-side deletion failure is discarded, so the confirmation closes without notifying the administrator that the member was not removed. \n The action invokes the asynchronous model method without attaching a rejection handler, leaving failures without user feedback.\n ",
+            "created_at": "2026-08-19T22:03:07Z"
+          },
+          {
+            "path": "app/assets/javascripts/admin/controllers/admin-group.js.es6",
+            "line": 64,
+            "body": "**CODE_MAINTAINABILTIY**: Attach a rejection handler to the promise returned by `addMembers` and surface the server error. Otherwise network or server-side failures leave the entered usernames in place without explaining that no members were added. \n The asynchronous add operation's return value is discarded, so rejected requests receive no user-facing handling.\n ",
+            "created_at": "2026-08-19T22:03:08Z"
+          },
+          {
+            "path": "app/controllers/admin/groups_controller.rb",
+            "line": 71,
+            "body": "**PERFORMANCE_OPTIMIZATION**: This loop calls `User.find_by_username` separately for every submitted username, producing one database lookup per member. Resolve the submitted usernames with a bulk query before adding the resulting users to the group. \n Adding N usernames currently performs N user SELECT queries instead of one bulk lookup.\n ",
+            "created_at": "2026-08-19T22:03:09Z"
+          },
+          {
+            "path": "app/controllers/groups_controller.rb",
+            "line": 22,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Clamp the caller-provided `limit` to a reasonable maximum. The current value is passed directly to the query, so a request such as `limit=100000` can fetch and serialize a very large group in one response. \n An unbounded SQL limit allows a single request to impose excessive database, memory, and serialization work.\n```\nlimit = [(params[:limit] || 50).to_i, 100].min\n``` ",
+            "created_at": "2026-08-19T22:03:10Z"
+          },
+          {
+            "path": "spec/controllers/admin/groups_controller_spec.rb",
+            "line": 115,
+            "body": "**ERROR**: Use `xhr :delete` here to match the declared `remove_member` route. Because this controller spec invokes the named action directly, the PUT request can pass without exercising the route's DELETE method contract. \n The test uses PUT while the production route exposes this action through DELETE, so it does not represent the routed request.\n```\nit \"cannot remove members from automatic groups\" do\n  xhr :delete, :remove_member, group_id: 1, user_id: 42\n  response.status.should == 422\nend\n``` ",
+            "created_at": "2026-08-19T22:03:11Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 060cda77729cb1c4a827560e09e89a7b22078ba9.",
+            "created_at": "2026-08-19T22:03:12Z"
+          }
+        ]
       }
     ]
   },
@@ -50865,6 +52051,43 @@
             "created_at": "2026-06-28T22:47:11Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "discourse__discourse-graphite__deputydev-v5__PR3__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR3__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "app/assets/javascripts/discourse/controllers/create_account_controller.js",
+            "line": 17,
+            "body": "**ERROR**: `rejectedEmails` is a mutable array initialized directly on the controller prototype, so separate controller instances share the same rejected-address state. An email pushed by one instance can therefore be rejected by another. Initialize a fresh `Em.A([])` for each instance in `init` instead. \n Mutating the prototype-level array with `pushObject` causes rejected emails to leak across controller instances.\n```\ninit: function() {\n  this._super();\n  this.set('rejectedEmails', Em.A([]));\n},\n``` ",
+            "created_at": "2026-08-19T21:37:57Z"
+          },
+          {
+            "path": "app/models/blocked_email.rb",
+            "line": 12,
+            "body": "**SECURITY**: The blocklist lookup compares the raw email string directly. With a case-sensitive comparison, a submitted address such as `BAD@spamclub.com` will not match a stored `bad@spamclub.com`, allowing the block to be bypassed. Normalize email casing consistently before lookup. \n Exact equality on an unnormalized email does not recognize differently cased forms of the same address under case-sensitive comparison.\n```\nnormalized = email.to_s.downcase.strip\nrecord = BlockedEmail.where(email: normalized).first\n``` ",
+            "created_at": "2026-08-19T21:37:58Z"
+          },
+          {
+            "path": "app/models/blocked_email.rb",
+            "line": 14,
+            "body": "**PERFORMANCE_OPTIMIZATION**: The counter update is a non-atomic read-modify-write. Two requests can both read `match_count = N`, increment to `N + 1`, and save the same value, losing one match. Use a single SQL update that increments `match_count` atomically while updating `last_match_at`. \n Concurrent calls can overwrite one another because the increment is calculated from a previously read value in Ruby.\n```\nBlockedEmail.where(id: record.id)\n  .update_all([\"match_count = match_count + 1, last_match_at = ?\", Time.zone.now])\n``` ",
+            "created_at": "2026-08-19T21:37:59Z"
+          },
+          {
+            "path": "lib/validators/email_validator.rb",
+            "line": 13,
+            "body": "**PERFORMANCE_OPTIMIZATION**: `BlockedEmail.should_block?` updates `match_count` and `last_match_at`, so invoking it from `validate_each` makes every validation mutate statistics. Because account creation can validate the same user before saving and again during `save`, one attempt can be counted more than once. Keep the validation check read-only and record the match once at the definitive enforcement point. \n Coupling the statistics write to model validation makes the count reflect validation invocations rather than actual registration attempts.\n```\ndef self.blocked?(email)\n  record = BlockedEmail.where(email: email).first\n  record && record.action_type == actions[:block]\nend\n\ndef self.record_match!(email)\n  BlockedEmail.where(email: email)\n    .update_all([\"match_count = match_count + 1, last_match_at = ?\", Time.zone.now])\nend\n``` ",
+            "created_at": "2026-08-19T21:38:00Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 5f8a130277dbddc95d133cd2832be639baf89213.",
+            "created_at": "2026-08-19T21:38:01Z"
+          }
+        ]
       }
     ]
   },
@@ -52089,6 +53312,43 @@
             "created_at": "2026-06-28T23:08:26Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "discourse__discourse-graphite__deputydev-v5__PR5__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR5__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "app/assets/stylesheets/common/base/topic-post.scss",
+            "line": 264,
+            "body": "**ERROR**: Converting `.small-action` to a flex container makes the existing `float: left` on its `.topic-avatar` child and `float: right` on its `button` child ineffective. Remove these dead declarations and, if the button must remain flush right, express that through the flex layout, such as `margin-left: auto`. \n Floats are ignored on direct flex items, so these declarations no longer control the avatar or button placement.\n```\n.small-action {\n  @include flexbox();\n  @include align-items(center);\n\n  .topic-avatar {\n    // Remove float: left\n  }\n\n  button {\n    margin-left: auto;\n    // Remove float: right\n  }\n}\n``` ",
+            "created_at": "2026-08-19T21:47:51Z"
+          },
+          {
+            "path": "app/assets/stylesheets/common/foundation/mixins.scss",
+            "line": 98,
+            "body": "**DOCUMENTATION**: The `//Flexbox` header is missing the space used by the other section comments in this file. Please change it to `// Flexbox` for consistency. \n The new section header does not follow the existing `// Text` comment format in this shared mixin file.\n```\n// Flexbox\n``` ",
+            "created_at": "2026-08-19T21:47:52Z"
+          },
+          {
+            "path": "app/assets/stylesheets/common/foundation/mixins.scss",
+            "line": 121,
+            "body": "**PERFORMANCE_OPTIMIZATION**: `-ms-align-items` is not a valid CSS property and will be ignored. The valid prefixed declaration, `-ms-flex-align`, is already present immediately above, so remove this dead declaration. \n Every use of this mixin emits an invalid vendor-prefixed property that provides no compatibility behavior.\n```\n@mixin align-items($alignment) {\n  -webkit-box-align: $alignment;\n  -webkit-align-items: $alignment;\n  -ms-flex-align: $alignment;\n  align-items: $alignment;\n}\n``` ",
+            "created_at": "2026-08-19T21:47:53Z"
+          },
+          {
+            "path": "app/assets/stylesheets/common/foundation/mixins.scss",
+            "line": 117,
+            "body": "**CODE_MAINTAINABILTIY**: The `align-items` mixin uses four-space indentation while the adjacent mixins use two spaces, and `align-items:$alignment` omits the space after the colon. Please format this block consistently with the rest of the file. \n The newly added mixin visibly departs from the formatting established by the surrounding mixins.\n```\n@mixin align-items($alignment) {\n  -webkit-box-align: $alignment;\n  -webkit-align-items: $alignment;\n  -ms-flex-align: $alignment;\n  align-items: $alignment;\n}\n``` ",
+            "created_at": "2026-08-19T21:47:54Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 5b229316ee4c661836ed1161139692a3e8527444.",
+            "created_at": "2026-08-19T21:47:55Z"
+          }
+        ]
       }
     ]
   },
@@ -53315,6 +54575,31 @@
             "created_at": "2026-06-28T23:23:17Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "discourse__discourse-graphite__deputydev-v5__PR6__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR6__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "app/serializers/user_serializer.rb",
+            "line": 138,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Parse the website URI once and reuse its host and path. On paths that append the path, the same URL is currently parsed once here and again in the selected branch. The sibling-subdomain logic likewise calls split repeatedly for both hosts; cache those component arrays before comparing their lengths and suffixes. \n Each serialization performs redundant parsing and allocates multiple equivalent host-component arrays from unchanged values.\n```\nuri = URI(website.to_s) rescue nil\nreturn if uri.nil? || uri.host.nil?\n\nwebsite_host = uri.host\nwebsite_path = uri.path\ndiscourse_host = Discourse.current_hostname\nwebsite_parts = website_host.split('.')\ndiscourse_parts = discourse_host.split('.')\n``` ",
+            "created_at": "2026-08-19T21:53:25Z"
+          },
+          {
+            "path": "spec/serializers/user_serializer_spec.rb",
+            "line": 76,
+            "body": "**CODE_MAINTAINABILTIY**: Add coverage for the equal-depth sibling-subdomain branch. The current examples exercise unrelated domains, an exact match, and a parent domain, but none enters the elsif at lines 144–146. Include cases where the suffix comparison both matches and does not match so both return paths are protected. \n Both outcomes of the newly introduced sibling-subdomain comparison are currently untested.\n ",
+            "created_at": "2026-08-19T21:53:26Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 267d8be1f556ed59639ced396c885bb44586da19.",
+            "created_at": "2026-08-19T21:53:27Z"
+          }
+        ]
       }
     ]
   },
@@ -55740,6 +57025,79 @@
             "created_at": "2026-06-28T23:03:36Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "discourse__discourse-graphite__deputydev-v5__PR4__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR4__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "app/assets/javascripts/embed.js",
+            "line": 17,
+            "body": "**ERROR**: Validate the message sender against the exact Discourse origin rather than using `discourseUrl.indexOf(e.origin)`. The current substring test accepts any serialized origin that happens to occur within `discourseUrl`; derive the origin from `discourseUrl` and require strict equality. \n A message from a non-equal origin can pass the substring check and set the iframe height.\n```\nvar allowedOrigin = new URL(discourseUrl).origin;\nif (e.origin !== allowedOrigin) { return; }\n``` ",
+            "created_at": "2026-08-19T21:42:47Z"
+          },
+          {
+            "path": "app/jobs/scheduled/poll_feed.rb",
+            "line": 35,
+            "body": "**CODE_MAINTAINABILTIY**: Guard `i.content` before calling `scrub`, or fall back to another item body field. Feed items may omit `content`, in which case this line raises `NoMethodError` and aborts processing of the remaining items. \n A content-less feed item terminates the polling run, and this job has retries disabled.\n```\ncontent = i.content || i.description || i.summary\nnext if content.blank?\ncontent = CGI.unescapeHTML(content.scrub)\nTopicEmbed.import(user, url, i.title, content)\n``` ",
+            "created_at": "2026-08-19T21:42:48Z"
+          },
+          {
+            "path": "app/jobs/scheduled/poll_feed.rb",
+            "line": 35,
+            "body": "**SECURITY**: Sanitize the feed item's HTML before passing it to `TopicEmbed.import`. `CGI.unescapeHTML` only decodes entities; the imported value is subsequently stored using `raw_html` with validations skipped and rendered with `raw`, so scripts or event-handler attributes from the feed can persist. \n Attacker-controlled HTML from the configured feed can be stored and rendered without a sanitization step.\n```\nsanitized_content = Sanitize.clean(CGI.unescapeHTML(i.content.scrub), Sanitize::Config::RELAXED)\nTopicEmbed.import(user, url, i.title, sanitized_content)\n``` ",
+            "created_at": "2026-08-19T21:42:49Z"
+          },
+          {
+            "path": "app/models/topic_embed.rb",
+            "line": 48,
+            "body": "**SECURITY**: Do not fetch `url` with unrestricted `open`. Validate the resolved destination and every redirect against private, loopback, and link-local ranges, restrict the scheme to HTTP/HTTPS, and enforce redirect limits before reading the response. \n The existing host check does not validate resolved addresses or redirect destinations, so the server-side fetch can reach an internal endpoint.\n```\nuri = URI.parse(url)\nraise ArgumentError, 'URL must be http/https' unless %w[http https].include?(uri.scheme)\n# Fetch with a client that rejects private/internal resolved IPs and revalidates each redirect.\n``` ",
+            "created_at": "2026-08-19T21:42:50Z"
+          },
+          {
+            "path": "app/models/topic_embed.rb",
+            "line": 13,
+            "body": "**SECURITY**: HTML-escape `url` before inserting it into the imported-from link's attribute and text. The HTTP-prefix check does not exclude quote characters, so the current interpolation can break out of the single-quoted `href`; this fragment is stored as raw HTML and later rendered with `raw`. \n A crafted URL can inject attributes or markup into content that is persisted and rendered without escaping.\n```\nescaped_url = ERB::Util.html_escape(url)\ncontents << \"\n<hr>\n<small>#{I18n.t('embed.imported_from', link: \"<a href='#{escaped_url}'>#{escaped_url}</a>\")}</small>\n\"\n``` ",
+            "created_at": "2026-08-19T21:42:51Z"
+          },
+          {
+            "path": "app/models/topic_embed.rb",
+            "line": 48,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Bound this remote read with explicit open/read timeouts and a maximum response size. `open(url).read` buffers the complete response before Readability processes it, allowing a slow or very large response to occupy the worker and consume excessive memory. \n An unbounded remote response can exhaust worker time and memory.\n```\nMAX_RESPONSE_BYTES = 1_048_576\nopen(url, read_timeout: 15, open_timeout: 10) do |response|\n  raise 'Response too large' if response.size && response.size > MAX_RESPONSE_BYTES\n  html = response.read(MAX_RESPONSE_BYTES)\nend\n``` ",
+            "created_at": "2026-08-19T21:42:52Z"
+          },
+          {
+            "path": "app/views/embed/best.html.erb",
+            "line": 6,
+            "body": "**CODE_MAINTAINABILTIY**: Replace `<%- end if %>` with `<%- end %>`. The trailing `if` has no modifier condition, so the generated Ruby is invalid and the embed template cannot render. \n Rendering this template raises a syntax error instead of returning the embedded topic view.\n```\n<%- end %>\n``` ",
+            "created_at": "2026-08-19T21:42:53Z"
+          },
+          {
+            "path": "app/views/layouts/embed.html.erb",
+            "line": 11,
+            "body": "**ERROR**: Do not interpolate the full `request.referer` directly into this JavaScript string. Extract its scheme/host/port for the `postMessage` target origin and JavaScript-escape the resulting value; a full path is not a valid target origin, and an unescaped quote in the referer can break out of the string. \n The current value can prevent resize-message delivery or inject JavaScript into the iframe response.\n```\n<% referer = URI(request.referer) %>\n<% target_origin = \"#{referer.scheme}://#{referer.host}#{referer.port && ![80, 443].include?(referer.port) ? \":#{referer.port}\" : \"\"}\" %>\nparent.postMessage({type: 'discourse-resize', height: document['body'].offsetHeight}, '<%= j(target_origin) %>');\n``` ",
+            "created_at": "2026-08-19T21:42:54Z"
+          },
+          {
+            "path": "lib/topic_retriever.rb",
+            "line": 27,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Set the throttle key and its TTL atomically. If the process stops after `setnx` succeeds but before `expire`, the key remains without an expiration and that URL is throttled permanently. \n The two-command sequence can leave a persistent throttle key after an interruption.\n```\nif $redis.set(retrieved_key, '1', nx: true, ex: 60)\n  return false\nend\n``` ",
+            "created_at": "2026-08-19T21:42:55Z"
+          },
+          {
+            "path": "lib/topic_retriever.rb",
+            "line": 41,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Avoid directly executing the scheduled `PollFeed` job from `TopicRetriever`. This call synchronously downloads and parses the entire feed and imports every item, blocking the retrieval worker for each unknown embed URL while feed polling is enabled. Let the scheduled job run independently or enqueue it instead of invoking `execute` inline. \n An on-demand retrieval miss triggers a complete network-bound feed poll and all associated database work in the same worker.\n```\nif SiteSetting.feed_polling_enabled?\n  Jobs.enqueue(:poll_feed)\n  return\nend\n\nfetch_http\n``` ",
+            "created_at": "2026-08-19T21:42:55Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 4f8aed295a29954023b2849c060ef4fb299d1b5d.",
+            "created_at": "2026-08-19T21:42:56Z"
+          }
+        ]
       }
     ]
   },
@@ -57329,6 +58687,37 @@
             "created_at": "2026-06-28T22:28:25Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "discourse__discourse-graphite__deputydev-v5__PR1__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR1__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "app/assets/javascripts/discourse/lib/utilities.js",
+            "line": 182,
+            "body": "**CODE_MAINTAINABILTIY**: Do not replace every type-specific upload limit with a uniform 10 MB value. This check previously used `max_<type>_size_kb`, so the change also relaxes client validation for attachment and avatar uploads that have lower configured limits. It also breaks the existing test that sets `max_image_size_kb` to 5 KB and expects a 10 KB file to be rejected. Preserve the per-type limits, or scope any relaxed allowance specifically to files eligible for server-side image downsizing and update the test contract accordingly. \n Configured per-type limits no longer drive client validation, so files above those limits are allowed through and the existing size-validation test no longer observes its expected alert.\n ",
+            "created_at": "2026-08-19T21:28:06Z"
+          },
+          {
+            "path": "app/controllers/uploads_controller.rb",
+            "line": 67,
+            "body": "**SECURITY**: Check the result of `OptimizedImage.downsize` and verify the final file size before calling `Upload.create_for`. `convert_with` returns `false` when ImageMagick fails, but this loop ignores that result and can launch all five conversions against an unchanged file. After the attempts are exhausted, execution still passes the file to `Upload.create_for` without confirming that it is now within `max_image_size_kb`. Stop retrying when conversion fails or makes no progress, and reject the upload if it remains oversized. \n A failed or insufficient conversion is treated as success, causing unnecessary ImageMagick work and allowing processing to continue with a file that remains above the intended limit.\n ",
+            "created_at": "2026-08-19T21:28:07Z"
+          },
+          {
+            "path": "app/models/optimized_image.rb",
+            "line": 149,
+            "body": "**SECURITY**: Ruby does not overload methods, so this second `self.downsize` definition replaces the preceding `(from, to, max_width, max_height, opts={})` definition. The existing `resize_emoji.rb` caller still passes five arguments as `downsize(path, path, 100, 100, opts)`, while the surviving method accepts at most four, causing `ArgumentError` whenever that job runs. Keep one signature and migrate the caller to pass `\"100x100\"` plus the options hash. \n The new definition changes the effective method arity without updating an existing caller, so emoji resize jobs fail at runtime.\n```\ndef self.downsize(from, to, dimensions, opts={})\n  optimize(\"downsize\", from, to, dimensions, opts)\nend\n\n# app/jobs/regular/resize_emoji.rb\nOptimizedImage.downsize(path, path, \"100x100\", opts)\n``` ",
+            "created_at": "2026-08-19T21:28:08Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit ffbaf8c54269df2ce510de91245760fddce09896.",
+            "created_at": "2026-08-19T21:28:09Z"
+          }
+        ]
       }
     ]
   },
@@ -58860,6 +60249,43 @@
             "created_at": "2026-06-28T22:28:38Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "discourse__discourse-graphite__deputydev-v5__PR2__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR2__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "app/assets/javascripts/discourse/controllers/topic-unsubscribe.js.es6",
+            "line": 5,
+            "body": "**CODE_MAINTAINABILTIY**: Rename `stopNotificiationsText` to `stopNotificationsText` here and in the Handlebars template. The current double-`i` misspelling is propagated across the controller/template binding and will become the property name consumers must use. \n The misspelled identifier is defined by the controller and referenced verbatim by the template, requiring a coordinated rename later.\n```\nstopNotificationsText: function() {\n  return I18n.t(\"topic.unsubscribe.stop_notifications\", { title: this.get(\"model.fancyTitle\") });\n}.property(\"model.fancyTitle\"),\n\n{{{stopNotificationsText}}}\n``` ",
+            "created_at": "2026-08-19T21:33:40Z"
+          },
+          {
+            "path": "app/controllers/topics_controller.rb",
+            "line": 105,
+            "body": "**ERROR**: Handle a missing `TopicUser` before reading `tu.notification_level`. `find_by` can return `nil` when this user/topic pair has no row, so the next line currently raises `NoMethodError` and returns a 500 response. \n The result of `TopicUser.find_by` is dereferenced unconditionally on the next line.\n```\ntu = TopicUser.find_by(user_id: current_user.id, topic_id: params[:topic_id])\n\nif tu.nil?\n  perform_show_response\n  return\nend\n\nif tu.notification_level > TopicUser.notification_levels[:regular]\n  tu.notification_level = TopicUser.notification_levels[:regular]\nelse\n  tu.notification_level = TopicUser.notification_levels[:muted]\nend\n\ntu.save!\n``` ",
+            "created_at": "2026-08-19T21:33:41Z"
+          },
+          {
+            "path": "app/views/email/notification.html.erb",
+            "line": 6,
+            "body": "**ERROR**: Remove this `%{respond_instructions}` footer from the `context_posts` block. The template already renders the same placeholder unconditionally at line 19, so emails containing context posts currently receive duplicate instructions. \n Both placeholder locations execute when `context_posts.present?`, producing duplicate email content.\n```\n<% if context_posts.present? %>\n  <hr>\n  <h4 class='.previous-discussion'><%= t \"user_notifications.previous_discussion\" %></h4>\n  <% context_posts.each do |p| %>\n    <%= render partial: 'email/post', locals: { post: p } %>\n  <% end %>\n<% end %>\n\n<hr>\n<div class='footer'>%{respond_instructions}</div>\n<div class='footer'>%{unsubscribe_link}</div>\n``` ",
+            "created_at": "2026-08-19T21:33:42Z"
+          },
+          {
+            "path": "config/routes.rb",
+            "line": 440,
+            "body": "**SECURITY**: Do not perform the unsubscribe mutation directly through GET. This action calls `tu.save!`, and the URL is included in notification emails, so a mail scanner, preview tool, or other automated link follower can change the user's notification level without user intent. Use GET only for a confirmation page and perform the mutation through POST or PATCH. \n An email-distributed GET endpoint persists a notification-state change whenever the link is followed.\n```\npost \"t/:slug/:topic_id/unsubscribe\" => \"topics#unsubscribe\", constraints: {topic_id: /\\d+/}\npost \"t/:topic_id/unsubscribe\" => \"topics#unsubscribe\", constraints: {topic_id: /\\d+/}\n``` ",
+            "created_at": "2026-08-19T21:33:43Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 6669a2d94d76eea3b99b8c476d12b1eb66726b07.",
+            "created_at": "2026-08-19T21:33:44Z"
+          }
+        ]
       }
     ]
   },
@@ -60473,6 +61899,43 @@
             "created_at": "2026-06-28T23:46:15Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR22532__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR22532__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "packages/app-store/googlecalendar/lib/CalendarService.ts",
+            "line": 1024,
+            "body": "**ERROR**: `updateManyByCredentialId` forwards this empty object directly to Prisma's `updateMany`, so no field is written and `@updatedAt` is not bumped. Pass an explicit `updatedAt` value so the cache-refresh timestamp is actually recorded. \n The empty update payload makes the new timestamp update a no-op, leaving SelectedCalendar.updatedAt stale after availability is refreshed.\n```\nawait SelectedCalendarRepository.updateManyByCredentialId(this.credential.id, {\n  updatedAt: new Date(),\n});\n``` ",
+            "created_at": "2026-08-19T23:56:59Z"
+          },
+          {
+            "path": "packages/features/apps/components/CredentialActionsDropdown.tsx",
+            "line": 42,
+            "body": "**CODE_MAINTAINABILTIY**: Invalidate `viewer.calendars.connectedCalendars` after deleting the cache. That query supplies `cacheUpdatedAt`; without invalidation, its cached result can continue showing the deleted cache timestamp, unlike the disconnect mutation below which explicitly invalidates the query. \n Deleting the backing cache does not refresh the tRPC query that provides the displayed cache status.\n```\nconst utils = trpc.useUtils();\nconst deleteCacheMutation = trpc.viewer.calendars.deleteCache.useMutation({\n  onSuccess: () => {\n    showToast(t(\"cache_deleted_successfully\"), \"success\");\n    onSuccess?.();\n  },\n  onError: () => {\n    showToast(t(\"error_deleting_cache\"), \"error\");\n  },\n  async onSettled() {\n    await utils.viewer.calendars.connectedCalendars.invalidate();\n  },\n});\n``` ",
+            "created_at": "2026-08-19T23:57:00Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/calendars/deleteCache.handler.ts",
+            "line": 25,
+            "body": "**ERROR**: Throw a `TRPCError` with an explicit `NOT_FOUND` or `UNAUTHORIZED` code here. A plain `Error` is reported by tRPC as `INTERNAL_SERVER_ERROR`, so a missing or inaccessible credential is incorrectly exposed to the client as a server failure. \n The ownership lookup's expected failure path currently produces incorrect HTTP and tRPC error semantics.\n```\nimport { TRPCError } from \"@trpc/server\";\n\n// ...\nif (!credential) {\n  throw new TRPCError({\n    code: \"NOT_FOUND\",\n    message: \"Credential not found or access denied\",\n  });\n}\n``` ",
+            "created_at": "2026-08-19T23:57:01Z"
+          },
+          {
+            "path": "scripts/test-gcal-webhooks.sh",
+            "line": 68,
+            "body": "**ERROR**: `sed -i ''` is BSD/macOS-specific and fails under GNU sed on Linux. Use platform-appropriate in-place syntax so updating an existing `GOOGLE_WEBHOOK_URL` does not terminate the script on Linux. \n The environment-file update uses an in-place sed argument form that GNU sed interprets incorrectly.\n```\nif sed --version >/dev/null 2>&1; then\n  sed -i -E \"s|^GOOGLE_WEBHOOK_URL=.*|GOOGLE_WEBHOOK_URL=$TUNNEL_URL|\" \"$ENV_FILE\"\nelse\n  sed -i '' -E \"s|^GOOGLE_WEBHOOK_URL=.*|GOOGLE_WEBHOOK_URL=$TUNNEL_URL|\" \"$ENV_FILE\"\nfi\n``` ",
+            "created_at": "2026-08-19T23:57:02Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 5fd11f9faa79c4aefd39975d1b0e963e5034f793.",
+            "created_at": "2026-08-19T23:57:03Z"
+          }
+        ]
       }
     ]
   },
@@ -61993,6 +63456,97 @@
             "created_at": "2026-06-28T23:11:51Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR8330__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR8330__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "packages/trpc/server/routers/viewer/slots.ts",
+            "line": 114,
+            "body": "**ERROR**: Dayjs object identity comparison instead of value comparison: `dayjs(date.start).add(utcOffset, 'minutes') === dayjs(date.end).add(utcOffset, 'minutes')` uses JavaScript's `===` operator to compare two distinct Dayjs wrapper objects. This always evaluates to `false` (unless they are the same object reference), so the zero-length override guard is unreachable. The intent is clearly to check if start equals end (a zero-duration override), which should use Dayjs's `.isSame()` method. \n JavaScript `===` on two different Dayjs object instances compares object identity, not timestamp value. Two `dayjs(...)` calls always produce different object references, so this condition will never be `true`. The zero-duration override is therefore silently ignored, meaning the surrounding block that is supposed to handle it (return `true` when start===end meaning slot is not in override window) never triggers. This is a semantic/logical error with direct booking correctness impact.\n```\nif (dayjs(date.start).add(utcOffset, \"minutes\").isSame(dayjs(date.end).add(utcOffset, \"minutes\"))) {\n  return true;\n}\n``` ",
+            "created_at": "2026-08-19T23:37:07Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/slots.ts",
+            "line": 142,
+            "body": "**ERROR**: Copy-paste error in working-hours end-time calculation: `const end = slotStartTime.hour() * 60 + slotStartTime.minute()` computes the slot's START time, not its end time. As a result, `start` and `end` hold the same value, and the condition `end > workingHour.endTime` always tests the start time against endTime instead of the actual slot end time. This means slots that would end outside working hours are never filtered, so bookings can be placed that run past the end of the organizer's working hours. \n Both `start` and `end` are calculated from `slotStartTime`, making them identical. The condition `end > workingHour.endTime` therefore never catches the case where the slot extends past the working-hour boundary. A slot starting at 17:45 for a 30-minute event (ending 18:15) in a schedule that closes at 18:00 would pass this guard incorrectly, allowing overbooking outside working hours.\n```\nconst start = slotStartTime.hour() * 60 + slotStartTime.minute();\nconst end = slotEndTime.hour() * 60 + slotEndTime.minute();\n``` ",
+            "created_at": "2026-08-19T23:37:08Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/slots.ts",
+            "line": 114,
+            "body": "**SECURITY**: Object identity comparison (`===`) used between two separate Dayjs instances to check for a zero-length date override. In JavaScript, two distinct objects are never `===` even if they represent the same time. This means the zero-length override branch (which should mark the slot as outside override) is unreachable, silently ignoring zero-length overrides. \n Using `===` to compare two `dayjs(...)` calls creates two separate object references that are never referentially equal, making the zero-length-override exclusion branch permanently unreachable. Correct comparison requires `.isSame()`.\n```\nif (dayjs(date.start).add(utcOffset, \"minutes\").isSame(dayjs(date.end).add(utcOffset, \"minutes\"))) {\n  return true;\n}\n``` ",
+            "created_at": "2026-08-19T23:37:09Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/slots.ts",
+            "line": 142,
+            "body": "**SECURITY**: In the new working-hours fallback check inside `checkIfIsAvailable`, the variable `end` is computed from `slotStartTime` instead of `slotEndTime`. Both `start` and `end` hold the same minute value (slot start, not slot end), so the check `end > workingHour.endTime` never detects slots whose end exceeds the working-hours window. Slots that extend past the end of a working-hours window are incorrectly treated as available. \n `const end = slotStartTime.hour() * 60 + slotStartTime.minute()` is an exact copy of the `start` expression—`slotEndTime` is not referenced. The boundary check `end > workingHour.endTime` therefore tests start-past-end, which is always false for valid slots, allowing over-length bookings past the working-hours window.\n```\nconst start = slotStartTime.hour() * 60 + slotStartTime.minute();\nconst end = slotEndTime.hour() * 60 + slotEndTime.minute();\nif (start < workingHour.startTime || end > workingHour.endTime) {\n  return true;\n}\n``` ",
+            "created_at": "2026-08-19T23:37:10Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/slots.ts",
+            "line": 79,
+            "body": "**SECURITY**: The flattened `dateOverrides` array (which includes entries tagged with `userId` for each host) is passed to every `checkIfIsAvailable` call, but the new date-override predicate inside the function does not filter overrides by the current user being checked. This means a date override belonging to host A can suppress or activate a time slot for host B in multi-host (e.g. round-robin or collective) events, producing incorrect availability results. \n The flattened `dateOverrides` at the outer scope (line 411-415) merges every user's overrides and is spread via `availabilityCheckProps` or passed directly through `...schedule`. The `checkIfIsAvailable` predicate iterates all of them without checking `userId`, so host A's override shifts/removes host B's availability window in collective and round-robin scenarios.\n```\n// In checkIfIsAvailable, filter dateOverrides for the relevant user before running the predicate:\n// When calling checkIfIsAvailable for a specific user (e.g., fixed host or loose host),\n// pass only that user's overrides:\nconst isAvailable = checkIfIsAvailable({\n  time: slot.time,\n  ...schedule,\n  ...availabilityCheckProps,\n  dateOverrides: schedule.dateOverrides,  // pass per-user overrides from 'schedule' directly, not the flattened global array\n  organizerTimeZone: schedule.timeZone,\n});\n\n// OR, inside checkIfIsAvailable, add a userId parameter and filter:\n// dateOverrides.filter(d => !d.userId || d.userId === currentUserId)\n``` ",
+            "created_at": "2026-08-19T23:37:11Z"
+          },
+          {
+            "path": "apps/web/test/lib/getSchedule.test.ts",
+            "line": 798,
+            "body": "**DOCUMENTATION**: The test assertion comment `// it should return the same as this is the utc time` is misleading. The test is checking that UTC slot times are the same regardless of the attendee's timezone (UTC+5:30 vs UTC+6:00), not simply because \"this is the utc time\". The comment fails to communicate the actual intent, which is that date override slot boundaries are stored/returned as UTC instants and should remain invariant no matter what the attendee's local timezone is. A reader unfamiliar with the fix may incorrectly interpret this as a no-op test. \n The comment at line +798 states 'it should return the same as this is the utc time', but this phrase is ambiguous and doesn't explain why the slots are identical across timezones. The test exists specifically to verify the timezone-offset bug fix (the entire PR motivation), yet its inline documentation doesn't capture that context. Future maintainers relying only on this comment would not understand what invariant is being tested.\n```\n// Both timezone variants (+5:30 and +6:00) should yield the same UTC slot times because\n// date override boundaries are stored and returned as UTC instants; the attendee's\n// local timezone does not shift the override window.\nexpect(scheduleForEventOnADayWithDateOverrideDifferentTimezone).toHaveTimeSlots(\n``` ",
+            "created_at": "2026-08-19T23:37:12Z"
+          },
+          {
+            "path": "packages/lib/slots.ts",
+            "line": 218,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Redundant dayjs object construction for override.start inside flatMap in getSlots. For each active override, `dayjs(override.start).utc().add(offset, \"minute\")` is constructed **twice** to compute `startTime`, and `dayjs(override.end).utc().add(offset, \"minute\")` is constructed **twice** to compute `endTime`. Each dayjs call parses, wraps, and adds, meaning 4 short-lived object allocations per override entry instead of 2. In a system processing many overrides for many users, across many days in the loop in `getSchedule`, this adds up unnecessarily. \n Each call to `dayjs(override.start).utc().add(offset, \"minute\")` allocates, parses, and chains a new Dayjs object. It is called twice for the same result when computing `startTime` (lines 219-220) and again twice for `endTime` (lines 222-223). Caching the intermediate Dayjs value eliminates 2 redundant object constructions per override per loop iteration in the day-level loop inside `getSchedule`.\n```\nconst overrides = activeOverrides.flatMap((override) => {\n  const organizerUtcOffset = dayjs(override.start.toString()).tz(override.timeZone).utcOffset();\n  const inviteeUtcOffset = dayjs(override.start.toString()).tz(timeZone).utcOffset();\n  const offset = inviteeUtcOffset - organizerUtcOffset;\n\n  const adjustedStart = dayjs(override.start).utc().add(offset, \"minute\");\n  const adjustedEnd = dayjs(override.end).utc().add(offset, \"minute\");\n\n  return {\n    userIds: override.userId ? [override.userId] : [],\n    startTime: adjustedStart.hour() * 60 + adjustedStart.minute(),\n    endTime: adjustedEnd.hour() * 60 + adjustedEnd.minute(),\n  };\n});\n``` ",
+            "created_at": "2026-08-19T23:37:12Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/slots.ts",
+            "line": 106,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Repeated redundant dayjs object constructions inside the `dateOverrides.find()` callback in `checkIfIsAvailable`. For every slot time check, `dayjs(date.start).add(utcOffset, \"minutes\")` is computed up to **3 times** and `dayjs(date.end).add(utcOffset, \"minutes\")` up to **2 times**, within a single `find()` iteration over all `dateOverrides`. Additionally, `dayjs.tz(date.start, organizerTimeZone).utcOffset()` is a timezone lookup called fresh on every iteration. Since `checkIfIsAvailable` is called in tight loops over `timeSlots` for each host in `getSchedule`, this results in a multiplicative allocation cost: O(slots × hosts × overrides) dayjs constructions. \n `checkIfIsAvailable` is called for every time slot for every host schedule (fixed, loose, and round-robin branches visible at lines 489, 514, 581 in the diff). Within each call the `dateOverrides.find()` loop recomputes `dayjs(date.start).add(utcOffset, \"minutes\")` three times and `dayjs(date.end).add(utcOffset, \"minutes\")` twice per override. This is O(slots × hosts × overrides) avoidable allocations. Caching these two adjusted values per iteration removes 3 redundant calls.\n```\nif (\n  dateOverrides.find((date) => {\n    const utcOffset = organizerTimeZone ? dayjs.tz(date.start, organizerTimeZone).utcOffset() * -1 : 0;\n    const adjustedStart = dayjs(date.start).add(utcOffset, \"minutes\");\n    const adjustedEnd = dayjs(date.end).add(utcOffset, \"minutes\");\n\n    if (adjustedStart.format(\"YYYY MM DD\") === slotStartTime.format(\"YYYY MM DD\")) {\n      dateOverrideExist = true;\n      if (adjustedStart.isSame(adjustedEnd)) {\n        return true;\n      }\n      if (slotEndTime.isBefore(adjustedStart) || slotEndTime.isSame(adjustedStart)) {\n        return true;\n      }\n      if (slotStartTime.isAfter(adjustedEnd)) {\n        return true;\n      }\n    }\n  })\n)\n``` ",
+            "created_at": "2026-08-19T23:37:13Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/slots.ts",
+            "line": 142,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Duplicate computation of `slotStartTime.hour() * 60 + slotStartTime.minute()` for both `start` and `end` in the working-hours fallback check inside `checkIfIsAvailable`. Both `start` and `end` variables are assigned the identical expression using `slotStartTime`, not `slotEndTime`. In addition to being a correctness bug (the slot-end time is never checked), this also means the same arithmetic is computed twice when it could be computed once and reused. \n Line 141 computes `start = slotStartTime.hour() * 60 + slotStartTime.minute()` and line 142 computes `end` with the same `slotStartTime` expression. Since `start === end` always, the condition `start < workingHour.startTime || end > workingHour.endTime` collapses to a single check, which means the slot-end-time boundary is never actually tested. Even correcting only the efficiency aspect, the second arithmetic computation is redundant when a single variable can be reused or when `slotEndTime` is used instead.\n```\nworkingHours.find((workingHour) => {\n  if (workingHour.days.includes(slotStartTime.day())) {\n    const start = slotStartTime.hour() * 60 + slotStartTime.minute();\n    const end = slotEndTime.hour() * 60 + slotEndTime.minute();\n    if (start < workingHour.startTime || end > workingHour.endTime) {\n      return true;\n    }\n  }\n})\n``` ",
+            "created_at": "2026-08-19T23:37:14Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/slots.ts",
+            "line": 114,
+            "body": "**CODE_MAINTAINABILTIY**: The equality check `dayjs(date.start).add(utcOffset, \"minutes\") === dayjs(date.end).add(utcOffset, \"minutes\")` compares two Dayjs object references, not their values. In JavaScript, two distinct object instances are never `===` to each other, so this branch is unreachable. The intent appears to be checking zero-length overrides where start equals end, but this will never be true, silently dropping the guard. \n The `===` operator on two separately-instantiated Dayjs objects compares object identity, not value. Dayjs `.isSame()` is the correct method. The unreachable branch means zero-length date overrides are silently treated as non-zero, potentially blocking valid slots or causing unexpected behavior.\n```\nif (dayjs(date.start).add(utcOffset, \"minutes\").isSame(dayjs(date.end).add(utcOffset, \"minutes\"))) {\n  return true;\n}\n``` ",
+            "created_at": "2026-08-19T23:37:15Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/slots.ts",
+            "line": 142,
+            "body": "**CODE_MAINTAINABILTIY**: In the working-hours fallback check, `const end` is calculated using `slotStartTime` instead of `slotEndTime`. Both `start` and `end` are assigned the same value (`slotStartTime.hour() * 60 + slotStartTime.minute()`), making the end check always compare the slot start against `workingHour.endTime`. As a result, any slot whose start falls within working hours but whose end exceeds the working-hours boundary will not be correctly excluded. \n `const end = slotStartTime.hour() * 60 + slotStartTime.minute()` is identical to `const start`. The variable `end` is supposed to represent the slot's end time in minutes-from-midnight, but it duplicates the start computation. This means slots whose end extends past working hours are incorrectly allowed, and the end boundary is never enforced.\n```\nconst start = slotStartTime.hour() * 60 + slotStartTime.minute();\nconst end = slotEndTime.hour() * 60 + slotEndTime.minute();\nif (start < workingHour.startTime || end > workingHour.endTime) {\n  return true;\n}\n``` ",
+            "created_at": "2026-08-19T23:37:16Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/slots.ts",
+            "line": 87,
+            "body": "**CODE_MAINTAINABILTIY**: In `checkIfIsAvailable`, the `dateOverrides` parameter does not include a `userId` field — the type only declares `{ start: Date; end: Date; }[]` — yet the flattened `dateOverrides` array passed from `getSchedule` carries per-user `userId` and `timeZone` properties. When `checkIfIsAvailable` is invoked with the full flattened array (containing all users' overrides), every override from every user is evaluated regardless of which host is being checked, meaning that one user's date override can make another user appear unavailable or available. The per-user filtering that is present in `getSlots` via `userIds` is absent from this predicate. \n The typed parameter `dateOverrides?: { start: Date; end: Date; }[]` discards the `userId` and `timeZone` fields present at runtime. Inside `checkIfIsAvailable`, the `find` callback iterates over all overrides without filtering by the current user being evaluated. In the fixed-host branch (`fixedHosts.every((schedule) => checkIfIsAvailable({ ...schedule, dateOverrides, ... }))`), the full multi-user `dateOverrides` array is passed, so a date override belonging to any other host may suppress the availability of the current fixed host.\n```\ndateOverrides?: {\n  start: Date;\n  end: Date;\n  userId?: number;\n  timeZone?: string;\n}[];\n\n// Then filter to current user's overrides before applying:\ndateOverrides.filter((d) => !d.userId || d.userId === schedule.user.id).find(...)\n``` ",
+            "created_at": "2026-08-19T23:37:16Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/slots.ts",
+            "line": 87,
+            "body": "**CODE_MAINTAINABILTIY**: The `dateOverrides` type declaration in `checkIfIsAvailable` lacks `timeZone` and `userId` fields, but the function body relies on `organizerTimeZone` (passed separately) for timezone conversion of those overrides. The `timeZone` property that was explicitly added to the flat `dateOverrides` array is not accessible inside the predicate function, making per-override timezone conversion impossible. This also means the type declaration is out of sync with the actual data shape passed at every call site. \n The `timeZone: availability.timeZone` field is explicitly added to each entry in the flattened `dateOverrides` array in `getSchedule`, but the `checkIfIsAvailable` parameter type does not declare this field. This means callers cannot leverage per-override timezone, the type is misleading, and future maintainers may remove the data at the source without knowing it was intended for the predicate.\n```\ndateOverrides?: {\n  start: Date;\n  end: Date;\n  userId?: number;\n  timeZone?: string;\n}[];\n``` ",
+            "created_at": "2026-08-19T23:37:17Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit ee38fd295fd294b9fc787eba482bde24bbfea69b.",
+            "created_at": "2026-08-19T23:37:18Z"
+          }
+        ]
       }
     ]
   },
@@ -63231,6 +64785,31 @@
             "created_at": "2026-06-28T23:31:37Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR14943__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR14943__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "packages/features/ee/workflows/api/scheduleSMSReminders.ts",
+            "line": 39,
+            "body": "**CODE_MAINTAINABILTIY**: Scope this retry-count branch to `method: WorkflowMethods.SMS`. As written, `deleteMany` deletes any `WorkflowReminder` with `retryCount > 1`, including reminders belonging to other delivery methods. \n The sibling branch and the subsequent scheduling query are SMS-specific, but this branch has no method constraint, allowing cross-method deletion.\n```\nawait prisma.workflowReminder.deleteMany({\n  where: {\n    OR: [\n      {\n        method: WorkflowMethods.SMS,\n        scheduledDate: {\n          lte: dayjs().toISOString(),\n        },\n      },\n      {\n        method: WorkflowMethods.SMS,\n        retryCount: {\n          gt: 1,\n        },\n      },\n    ],\n  },\n});\n``` ",
+            "created_at": "2026-08-19T23:46:14Z"
+          },
+          {
+            "path": "packages/features/ee/workflows/api/scheduleSMSReminders.ts",
+            "line": 184,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Use Prisma's atomic increment operation here and in the catch path. `reminder.retryCount + 1` is based on the value fetched before the loop, so overlapping cron executions can read the same count and overwrite each other with the same result. \n A lost increment delays retry-based cleanup because concurrent executions can both write the same stale value instead of recording both failures.\n```\ndata: {\n  retryCount: { increment: 1 },\n}\n``` ",
+            "created_at": "2026-08-19T23:46:15Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit c790227e0cda780e6fea9bb03af27948d9e286b9.",
+            "created_at": "2026-08-19T23:46:16Z"
+          }
+        ]
       }
     ]
   },
@@ -64407,6 +65986,31 @@
             "created_at": "2026-06-28T22:40:20Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR22345__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR22345__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "packages/lib/server/service/insightsBooking.ts",
+            "line": 72,
+            "body": "**ERROR**: `getAuthorizationConditions()` returns a non-nullable `Prisma.Sql`, so `authConditions` is always truthy. Consequently, the filter-only and final `NOTHING_CONDITION` branches here are unreachable. Simplify this to branch only on `filterConditions`: combine it with authorization when present, otherwise return the authorization condition directly. \n The current control flow advertises filter-only and neither-condition states that cannot occur under the method's declared contract.\n```\nasync getBaseConditions(): Promise<Prisma.Sql> {\n  const authConditions = await this.getAuthorizationConditions();\n  const filterConditions = await this.getFilterConditions();\n\n  if (filterConditions) {\n    return Prisma.sql`(${authConditions}) AND (${filterConditions})`;\n  }\n  return authConditions;\n}\n``` ",
+            "created_at": "2026-08-19T23:28:45Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "**CODE_MAINTAINABILTIY**: Please retain regression coverage for caching. This removes both caching tests, but `InsightsBookingService` still memoizes authorization and filter conditions using `cachedAuthConditions` and `cachedFilterConditions`. Restore equivalent tests for the new `Prisma.Sql` results and use strict reference equality on consecutive calls to verify that the cached object is returned. \n Without these tests, a change that rebuilds conditions on every call instead of returning the memoized value would no longer be detected.\n```\nconst conditions1 = await service.getAuthorizationConditions();\nconst conditions2 = await service.getAuthorizationConditions();\nexpect(conditions2).toBe(conditions1);\n\nconst filterConditions1 = await service.getFilterConditions();\nconst filterConditions2 = await service.getFilterConditions();\nexpect(filterConditions2).toBe(filterConditions1);\n``` ",
+            "created_at": "2026-08-19T23:28:44Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit da09bc0808e7c6df1bc10e00d5acf764d94c7cc7.",
+            "created_at": "2026-08-19T23:28:46Z"
+          }
+        ]
       }
     ]
   },
@@ -66706,6 +68310,91 @@
             "created_at": "2026-06-28T23:17:49Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR11059__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR11059__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "apps/web/pages/api/webhook/app-credential.ts",
+            "line": 34,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Run the user and app lookups concurrently. Neither query depends on the other, but the current sequential awaits add a second database round-trip before every credential sync can continue. \n The independent user and app existence checks incur serial database latency on every webhook invocation.\n```\nconst [user, app] = await Promise.all([\n  prisma.user.findUnique({ where: { id: reqBody.userId } }),\n  prisma.app.findUnique({\n    where: { slug: reqBody.appSlug },\n    select: { slug: true },\n  }),\n]);\n``` ",
+            "created_at": "2026-08-19T23:42:55Z"
+          },
+          {
+            "path": "apps/web/pages/api/webhook/app-credential.ts",
+            "line": 31,
+            "body": "**ERROR**: Handle request-body validation explicitly. `schema.parse(req.body)` throws when the body is malformed or missing, so an ordinary client validation failure currently escapes the handler as a 500 instead of returning a controlled 400 response. \n Malformed webhook bodies trigger an unhandled Zod exception rather than the handler's structured 4xx response flow.\n```\nconst parsedBody = appCredentialWebhookRequestBodySchema.safeParse(req.body);\nif (!parsedBody.success) {\n  return res.status(400).json({ message: \"Invalid request body\" });\n}\nconst reqBody = parsedBody.data;\n``` ",
+            "created_at": "2026-08-19T23:42:56Z"
+          },
+          {
+            "path": "apps/web/pages/api/webhook/app-credential.ts",
+            "line": 57,
+            "body": "**SECURITY**: Validate the decrypted credential payload before persisting it. This `JSON.parse` can throw for malformed decrypted data, and any successfully parsed JSON value is currently written directly to `credential.key` without checking the structure expected by the selected app. \n Malformed JSON causes an uncontrolled exception, while structurally invalid JSON can be persisted as an app credential.\n ",
+            "created_at": "2026-08-19T23:42:57Z"
+          },
+          {
+            "path": "packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts",
+            "line": 5,
+            "body": "**CODE_MAINTAINABILTIY**: These computed keys do not define wildcard properties in a Zod object. Calling `toString()` produces fixed JavaScript property names, so the schema does not validate arbitrary numeric expiry fields or preserve arbitrary token fields as the comments intend. Use explicit properties plus `passthrough`/`catchall`, with a refinement if a numeric expiry is required. \n The sync-mode schema does not implement its intended dynamic token-response validation.\n```\nconst minimumTokenResponseSchema = z\n  .object({\n    access_token: z.string(),\n  })\n  .passthrough();\n``` ",
+            "created_at": "2026-08-19T23:42:58Z"
+          },
+          {
+            "path": "packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts",
+            "line": 26,
+            "body": "**ERROR**: Do not replace an omitted refresh token with the literal string `\"refresh_token\"`. Callers that persist the returned value can overwrite the valid stored token with this placeholder, causing the next provider refresh to fail. Preserve the existing token at the caller or reject the response instead. \n The placeholder can be persisted as an actual OAuth refresh token and subsequently rejected by the provider.\n```\nif (!refreshTokenResponse.data.refresh_token) {\n  throw new Error(\"Refresh token is missing from the token response\");\n}\n``` ",
+            "created_at": "2026-08-19T23:42:59Z"
+          },
+          {
+            "path": "packages/app-store/_utils/oauth/refreshOAuthTokens.ts",
+            "line": 8,
+            "body": "**SECURITY**: Authenticate requests to the credential-sync endpoint. This POST identifies a Cal.com user and app but currently sends no authorization or shared-secret information, leaving the external endpoint without a way to verify that the request originated from this service. \n The credential-sync request has no application-level authentication mechanism.\n ",
+            "created_at": "2026-08-19T23:43:00Z"
+          },
+          {
+            "path": "packages/app-store/_utils/oauth/refreshOAuthTokens.ts",
+            "line": 8,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Bound this external request with an abort timeout. The fetch is awaited directly on the OAuth refresh path, so a credential-sync endpoint that stops responding can leave the associated calendar or video operation waiting indefinitely. \n An unresponsive external sync service can indefinitely stall token refreshes that await this fetch.\n```\nconst controller = new AbortController();\nconst timeoutId = setTimeout(() => controller.abort(), 10_000);\ntry {\n  const response = await fetch(process.env.CALCOM_CREDENTIAL_SYNC_ENDPOINT, {\n    method: \"POST\",\n    body: new URLSearchParams({ calcomUserId: userId.toString(), appSlug }),\n    signal: controller.signal,\n  });\n  return response;\n} finally {\n  clearTimeout(timeoutId);\n}\n``` ",
+            "created_at": "2026-08-19T23:43:01Z"
+          },
+          {
+            "path": "packages/app-store/_utils/oauth/refreshOAuthTokens.ts",
+            "line": 14,
+            "body": "**CODE_MAINTAINABILTIY**: Normalize the sync branch to the same contract expected by each caller instead of returning a raw Fetch `Response`. For example, Zoho Bigin immediately reads `.data`, while HubSpot treats the result as a `HubspotToken`; neither shape exists on this `Response`, so sync-mode refreshes produce undefined fields or runtime errors. \n The helper changes return type in sync mode without adapting callers that expect provider-specific token objects.\n ",
+            "created_at": "2026-08-19T23:43:01Z"
+          },
+          {
+            "path": "packages/app-store/_utils/oauth/refreshOAuthTokens.ts",
+            "line": 14,
+            "body": "**CODE_MAINTAINABILTIY**: Check `response.ok` before returning the credential-sync response. Fetch resolves normally for 4xx and 5xx statuses, so these failures currently continue into callers as if token refresh had succeeded and only fail later while parsing or using the response. \n HTTP error responses from the external sync endpoint are not converted into refresh failures.\n```\nif (!response.ok) {\n  throw new Error(`Credential sync endpoint returned ${response.status}: ${response.statusText}`);\n}\n``` ",
+            "created_at": "2026-08-19T23:43:03Z"
+          },
+          {
+            "path": "packages/app-store/googlecalendar/lib/CalendarService.ts",
+            "line": 97,
+            "body": "**CODE_MAINTAINABILTIY**: Store the parsed credential data rather than the `safeParse` result wrapper. `parseRefreshTokenResponse` returns `{ success: true, data: ... }`, but this value is assigned directly to `key`, so Prisma receives the wrapper instead of the Google credential object. \n Google persists the validation wrapper as its credential key, changing the stored credential shape.\n```\nconst parsedKey = parseRefreshTokenResponse(googleCredentials, googleCredentialSchema);\nawait prisma.credential.update({\n  where: { id: credential.id },\n  data: { key: parsedKey.data },\n});\n``` ",
+            "created_at": "2026-08-19T23:43:04Z"
+          },
+          {
+            "path": "packages/app-store/salesforce/lib/CalendarService.ts",
+            "line": 92,
+            "body": "**ERROR**: Remove this unreachable failure branch or change the helper's contract. `parseRefreshTokenResponse` already throws when validation fails, so any value returned here necessarily has `success === true`; this check can never execute. \n The Salesforce code handles a failed result that the parsing helper never returns.\n```\nconst accessTokenParsed = parseRefreshTokenResponse(accessTokenJson, salesforceTokenSchema);\n\nawait prisma.credential.update({\n  where: { id: credential.id },\n  data: { key: { ...accessTokenParsed.data, refresh_token: credentialKey.refresh_token } },\n});\n``` ",
+            "created_at": "2026-08-19T23:43:05Z"
+          },
+          {
+            "path": "packages/app-store/zoho-bigin/lib/CalendarService.ts",
+            "line": 93,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Pass the credential owner's user ID here, not `credentialId`. The helper serializes its third argument as `calcomUserId`, but this method's `credentialId` is the credential row's primary key, so sync-mode requests identify the wrong user. \n Zoho Bigin sends a credential database ID in the payload field reserved for a Cal.com user ID.\n ",
+            "created_at": "2026-08-19T23:43:06Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 9fde0e906897cc0f4f71793f647dd629faba3317.",
+            "created_at": "2026-08-19T23:43:07Z"
+          }
+        ]
       }
     ]
   },
@@ -68422,6 +70111,49 @@
             "created_at": "2026-06-28T23:04:20Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR7232__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR7232__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "packages/features/bookings/lib/handleCancelBooking.ts",
+            "line": 485,
+            "body": "**CODE_MAINTAINABILTIY**: Await these reminder cancellations instead of starting them inside `forEach`. Both helpers perform asynchronous provider or database work, but their promises are discarded here, so the cancellation handler can continue and respond before the reminder rows have been cancelled or deleted. Build promises with `map`/`flatMap` and await them with `Promise.all`. \n The nested `forEach` loops discard every deletion promise, allowing booking cancellation to complete before reminder cancellation finishes.\n```\nawait Promise.all(\n  updatedBookings.flatMap((booking) =>\n    booking.workflowReminders.map((reminder) => {\n      if (reminder.method === WorkflowMethods.EMAIL) {\n        return deleteScheduledEmailReminder(reminder.id, reminder.referenceId);\n      }\n      if (reminder.method === WorkflowMethods.SMS) {\n        return deleteScheduledSMSReminder(reminder.id, reminder.referenceId);\n      }\n      return Promise.resolve();\n    })\n  )\n);\n``` ",
+            "created_at": "2026-08-19T23:33:09Z"
+          },
+          {
+            "path": "packages/features/bookings/lib/handleNewBooking.ts",
+            "line": 966,
+            "body": "**ERROR**: Await the reminder-cancellation promises before proceeding with the reschedule. `forEach` discards the promises returned by these helpers, so the surrounding `try/catch` ends immediately and the reschedule can continue before the previous booking's reminders are cancelled. Use `map` plus `await Promise.all(...)` inside this `try` block. \n The reschedule proceeds concurrently with cancellation of the old reminders, leaving a window in which those reminders remain active.\n```\ntry {\n  const cancellationPromises = originalRescheduledBooking.workflowReminders.map((reminder) => {\n    if (reminder.method === WorkflowMethods.EMAIL) {\n      return deleteScheduledEmailReminder(reminder.id, reminder.referenceId, true);\n    }\n    if (reminder.method === WorkflowMethods.SMS) {\n      return deleteScheduledSMSReminder(reminder.id, reminder.referenceId);\n    }\n    return Promise.resolve();\n  });\n  await Promise.all(cancellationPromises);\n} catch (error) {\n  log.error(\"Error while canceling scheduled workflow reminders\", error);\n}\n``` ",
+            "created_at": "2026-08-19T23:33:10Z"
+          },
+          {
+            "path": "packages/features/ee/workflows/api/scheduleEmailReminders.ts",
+            "line": 53,
+            "body": "**ERROR**: Handle cancellation failures per reminder rather than wrapping the entire loop in one `try/catch`. If any `client.request` rejects, control jumps out of the loop, all later reminders remain active, and `Promise.all(workflowRemindersToDelete)` is never reached even for cancellations that already succeeded. Catch each iteration independently and delete that reminder after its provider cancellation succeeds so one bad batch does not block the rest. \n One SendGrid failure aborts the complete batch and abandons both subsequent cancellations and previously accumulated database deletions.\n```\nfor (const reminder of remindersToCancel) {\n  try {\n    await client.request({\n      url: \"/v3/user/scheduled_sends\",\n      method: \"POST\",\n      body: {\n        batch_id: reminder.referenceId,\n        status: \"cancel\",\n      },\n    });\n    await prisma.workflowReminder.delete({\n      where: { id: reminder.id },\n    });\n  } catch (error) {\n    console.log(`Error cancelling scheduled Email reminder ${reminder.id}: ${error}`);\n  }\n}\n``` ",
+            "created_at": "2026-08-19T23:33:11Z"
+          },
+          {
+            "path": "packages/features/ee/workflows/components/WorkflowStepContainer.tsx",
+            "line": 439,
+            "body": "**CODE_MAINTAINABILTIY**: Hard-coded UI strings `\"Verification code\"` and `\"Verify\"` are used in the phone verification section of `WorkflowStepContainer`. The rest of the component uses the `t(...)` i18n function for all user-facing text. These two strings bypass the translation system and will always appear in English regardless of the user's locale. \n The placeholder and button label are bare English literals in an otherwise localized component.\n```\n<TextField\n  placeholder={t(\"verification_code\")}\n  ...\n/>\n<Button ...>\n  {t(\"verify\")}\n</Button>\n``` ",
+            "created_at": "2026-08-19T23:33:12Z"
+          },
+          {
+            "path": "packages/features/ee/workflows/lib/reminders/emailReminderManager.ts",
+            "line": 222,
+            "body": "**ERROR**: The `immediateDelete` branch cancels the SendGrid batch and then returns without deleting or updating the local `workflowReminder` row. Callers using this mode therefore leave stale reminder records until some later cleanup can remove them. Delete the local row after the provider cancellation succeeds, as the null-reference and SMS deletion paths do. \n Returning immediately after the SendGrid request leaves the database record in its original state despite an immediate deletion request.\n```\nif (immediateDelete) {\n  await client.request({\n    url: \"/v3/user/scheduled_sends\",\n    method: \"POST\",\n    body: {\n      batch_id: referenceId,\n      status: \"cancel\",\n    },\n  });\n  await prisma.workflowReminder.delete({\n    where: { id: reminderId },\n  });\n  return;\n}\n``` ",
+            "created_at": "2026-08-19T23:33:13Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 6048e2a86b50e81e1e3b1b467dfea5a895add3dc.",
+            "created_at": "2026-08-19T23:33:14Z"
+          }
+        ]
       }
     ]
   },
@@ -70394,6 +72126,55 @@
             "created_at": "2026-06-28T23:39:36Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR14740__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR14740__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "apps/web/components/dialog/AddGuestsDialog.tsx",
+            "line": 26,
+            "body": "**CODE_MAINTAINABILTIY**: Hoist `ZAddGuestsInputSchema` to module scope. It has no dependency on component state or props, so defining it here unnecessarily creates a new schema object on every render. \n Every component render reconstructs the same static Zod schema.\n```\nconst ZAddGuestsInputSchema = z.array(z.string().email()).refine((emails) => {\n  const uniqueEmails = new Set(emails);\n  return uniqueEmails.size === emails.length;\n});\n\nexport const AddGuestsDialog = (props: IAddGuestsDialog) => {\n  // Component implementation\n};\n``` ",
+            "created_at": "2026-08-19T23:51:12Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+            "line": 47,
+            "body": "**ERROR**: Use `||` between these permission checks. `isTeamAdmin` accepts ADMIN and OWNER roles, while `isTeamOwner` accepts only OWNER; combining them with `&&` therefore denies ADMIN-only members even though this pathway is intended for an admin or owner. \n An ADMIN who is neither the organizer nor an attendee fails this authorization check and cannot add guests.\n```\nconst isTeamAdminOrOwner =\n  (await isTeamAdmin(user.id, booking.eventType?.teamId ?? 0)) ||\n  (await isTeamOwner(user.id, booking.eventType?.teamId ?? 0));\n``` ",
+            "created_at": "2026-08-19T23:51:14Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+            "line": 77,
+            "body": "**ERROR**: Normalize `guest` before both comparisons. The blacklist is stored in lowercase but checked against the original casing, and the attendee comparison is also case-sensitive, so case variants can bypass the blacklist or be inserted as duplicate attendees. \n For example, an uppercase variant of a lowercased blacklist entry passes this filter, as does a case variant of an existing attendee address.\n```\nconst uniqueGuests = guests.filter(\n  (guest) =>\n    !booking.attendees.some((attendee) => guest.toLowerCase() === attendee.email.toLowerCase()) &&\n    !blacklistedGuestEmails.includes(guest.toLowerCase())\n);\n``` ",
+            "created_at": "2026-08-19T23:51:14Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+            "line": 120,
+            "body": "**PERFORMANCE_OPTIMIZATION**: Start the organizer translation together with the attendee translations. These operations are independent, but the current second `await` does not begin until every attendee translation has resolved, adding one avoidable serial translation wait. \n Calendar processing is delayed by the organizer translation latency after the attendee translation batch has already completed.\n```\nconst [attendeesList, tOrganizer] = await Promise.all([\n  Promise.all(attendeesListPromises),\n  getTranslation(organizer.locale ?? \"en\", \"common\"),\n]);\n``` ",
+            "created_at": "2026-08-19T23:51:15Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+            "line": 168,
+            "body": "**CODE_MAINTAINABILTIY**: Pass `uniqueGuests` to `sendAddGuestsEmails`, since that is the set actually inserted. The email manager uses this argument to identify newly added attendees; passing the original input can classify an already-existing submitted attendee as new and select the full booking-confirmation template instead of the guests-added notification. \n The database insertion and email-template routing currently use different definitions of which guests were newly added.\n```\ntry {\n  await sendAddGuestsEmails(evt, uniqueGuests);\n} catch (err) {\n  console.log(\"Error sending AddGuestsEmails\");\n}\n``` ",
+            "created_at": "2026-08-19T23:51:16Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+            "line": 170,
+            "body": "**DOCUMENTATION**: Log the caught email-delivery error rather than discarding it. This context-free `console.log` records neither the error nor the affected booking or user, so a delivery failure cannot be diagnosed from the emitted entry. \n The handler swallows the email exception and leaves no stack, error details, or request context in the log.\n```\nimport logger from \"@calcom/lib/logger\";\n\n// ...\ntry {\n  await sendAddGuestsEmails(evt, guests);\n} catch (err) {\n  logger.error(\"Error sending AddGuestsEmails\", { bookingId, userId: user.id, err });\n}\n``` ",
+            "created_at": "2026-08-19T23:51:17Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 92f44dcea7ff19e9123a30c63c167a2938df5a55.",
+            "created_at": "2026-08-19T23:51:18Z"
+          }
+        ]
       }
     ]
   },
@@ -72114,6 +73895,79 @@
             "created_at": "2026-06-28T22:29:53Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR10600__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR10600__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "apps/web/components/settings/EnableTwoFactorModal.tsx",
+            "line": 278,
+            "body": "**ERROR**: The `navigator.clipboard.writeText()` call is not awaited, so the success toast (`showToast`) is shown regardless of whether the clipboard write actually succeeded. Additionally, `navigator.clipboard` is not available in non-HTTPS contexts and the rejected promise will be silently swallowed since there is no `.catch()` handler. Users on HTTP or those who deny clipboard permission will see a false-success toast. \n The promise from `navigator.clipboard.writeText()` is not awaited and has no `.catch()` handler. When the clipboard API is unavailable (non-secure context or user denies permission), the error is silently swallowed and the success toast is still shown — giving the user false confidence that codes were copied.\n```\nonClick={(e) => {\n  e.preventDefault();\n  navigator.clipboard.writeText(backupCodes.map(formatBackupCode).join(\"\n\"))\n    .then(() => showToast(t(\"backup_codes_copied\"), \"success\"))\n    .catch(() => showToast(t(\"something_went_wrong\"), \"error\"));\n}}\n``` ",
+            "created_at": "2026-08-19T23:18:28Z"
+          },
+          {
+            "path": "packages/features/auth/lib/next-auth-options.ts",
+            "line": 131,
+            "body": "**ERROR**: In `next-auth-options.ts`, the password verification is skipped when `credentials.totpCode` is set (the condition is `if (user.password && !credentials.totpCode)`). This means that when a user submits a `backupCode` credential, `credentials.totpCode` is empty/undefined, so the password check **is** executed. However, this creates a subtle logical issue: the password check gate `!credentials.totpCode` is actually checking for TOTP code presence rather than backup code presence. If a user sends both `totpCode` AND `backupCode` credentials simultaneously, the password check is entirely skipped (`!credentials.totpCode` is false), yet neither the TOTP nor backup code branches have the password verified. A user could craft a login request with both fields set and skip password verification. \n The condition `if (user.password && !credentials.totpCode)` evaluates to `false` (skipping password verification) whenever `credentials.totpCode` is non-empty. If an attacker submits both `totpCode` and `backupCode` simultaneously, the totpCode causes the password check to be skipped, the backup code then grants access. This is a logical bypass: the password verification gate should apply unless the user is in a recognized 2FA continuation flow.\n```\n// Replace the password check condition to also skip when backupCode is provided:\nif (user.password && !credentials.totpCode && !credentials.backupCode) {\n  if (!user.password) {\n    throw new Error(ErrorCode.IncorrectEmailPassword);\n  }\n  const isCorrectPassword = await verifyPassword(credentials.password, user.password);\n  if (!isCorrectPassword) {\n    throw new Error(ErrorCode.IncorrectEmailPassword);\n  }\n}\n``` ",
+            "created_at": "2026-08-19T23:18:29Z"
+          },
+          {
+            "path": "apps/web/components/auth/BackupCode.tsx",
+            "line": 7,
+            "body": "**DOCUMENTATION**: The exported function in BackupCode.tsx is named `TwoFactor` but the file is `BackupCode.tsx`. The function name mismatch is confusing and does not communicate the component's actual purpose to future readers. \n The file is named `BackupCode.tsx` and is imported everywhere as `BackupCode`, but its internal function declaration is `TwoFactor`. Any developer reading the component definition alone cannot understand what it does without reading its body. The mismatch also defeats IDE name-resolution and makes stack traces ambiguous.\n```\nexport default function BackupCode({ center = true }) {\n``` ",
+            "created_at": "2026-08-19T23:18:31Z"
+          },
+          {
+            "path": "apps/web/pages/api/auth/two-factor/totp/disable.ts",
+            "line": 50,
+            "body": "**DOCUMENTATION**: The `console.error` message in the backup-code login path of `disable.ts` says \"cannot proceed with backup code login\" but the context is a 2FA-disable operation, not a login. The log message will mislead anyone triaging errors from this endpoint. \n The log says 'backup code login' but is in the disable-2FA endpoint. During incident triage a support engineer searching logs for 'login' will find this message but be looking at the wrong endpoint, adding confusion and slowing diagnosis.\n```\nconsole.error(\"Missing encryption key; cannot proceed with backup code verification for 2FA disable.\");\n``` ",
+            "created_at": "2026-08-19T23:18:32Z"
+          },
+          {
+            "path": "apps/web/components/settings/EnableTwoFactorModal.tsx",
+            "line": 99,
+            "body": "**PERFORMANCE_OPTIMIZATION**: The `EnableTwoFactorModal` creates a Blob object URL (`URL.createObjectURL`) for backup code download each time `handleSetup` succeeds, but the URL is only revoked on the *next* setup attempt (`if (backupCodesUrl) URL.revokeObjectURL(backupCodesUrl)` before the new one is created). The URL is never revoked when the modal is closed normally (via `onOpenChange`, Cancel, or the Close button in backup codes step), nor when `resetState()` is called. This leaks an object URL for the lifetime of the page, holding the text Blob in memory. \n Blob URLs created with `URL.createObjectURL` hold a reference to the underlying Blob in memory until explicitly revoked with `URL.revokeObjectURL`. The current code only revokes on the next successful setup call (line 98), but not when the modal is closed via `onOpenChange`, when the user clicks Cancel, or when `resetState()` is invoked from the Close button. This means the Blob and its data remain in memory for the page lifetime, accumulating on each modal open without a full page navigation.\n```\n// In EnableTwoFactorModal, add a useEffect cleanup:\nuseEffect(() => {\n  return () => {\n    if (backupCodesUrl) {\n      URL.revokeObjectURL(backupCodesUrl);\n    }\n  };\n}, [backupCodesUrl]);\n\n// Also update resetState to revoke:\nconst resetState = () => {\n  setPassword(\"\");\n  setErrorMessage(null);\n  setStep(SetupStep.ConfirmPassword);\n  if (backupCodesUrl) {\n    URL.revokeObjectURL(backupCodesUrl);\n    setBackupCodesUrl(\"\");\n  }\n};\n``` ",
+            "created_at": "2026-08-19T23:18:33Z"
+          },
+          {
+            "path": "apps/web/components/settings/TwoFactorAuthAPI.ts",
+            "line": 22,
+            "body": "**CODE_MAINTAINABILTIY**: The `apps/web/components/security/DisableTwoFactorModal.tsx` file is a duplicate of `apps/web/components/settings/DisableTwoFactorModal.tsx` and was NOT updated in this PR. It still calls `TwoFactorAuthAPI.disable(password, totpCode)` with 2 arguments, while the updated API in `apps/web/components/settings/TwoFactorAuthAPI.ts` now expects 3 arguments. Additionally, `apps/web/components/security/TwoFactorAuthAPI.ts` still has the old 2-argument `disable` signature.\n\nWhenever `apps/web/components/security/DisableTwoFactorModal.tsx` is rendered (e.g., in a different settings page/section), users will not be able to use backup codes to disable 2FA. This duplicated code creates an ongoing maintenance burden where security-critical changes must be applied to multiple copies. \n The `disable` function signature is changed only in `apps/web/components/settings/TwoFactorAuthAPI.ts`. There is an identical but untouched duplicate at `apps/web/components/security/TwoFactorAuthAPI.ts` (line 22) and its paired `DisableTwoFactorModal.tsx` (line 38) that still use the old 2-argument call. The backup code flow will silently never send a `backupCode` parameter when that component is rendered, making backup-code disable nonfunctional for that code path.\n```\n// In apps/web/components/security/TwoFactorAuthAPI.ts, update the disable method:\nasync disable(password: string, code: string, backupCode?: string) {\n  return fetch(\"/api/auth/two-factor/totp/disable\", {\n    method: \"POST\",\n    body: JSON.stringify({ password, code, backupCode }),\n    headers: {\n      \"Content-Type\": \"application/json\",\n    },\n  });\n},\n\n// Consider consolidating both DisableTwoFactorModal.tsx files into a single shared component\n// to avoid diverging implementations.\n``` ",
+            "created_at": "2026-08-19T23:18:34Z"
+          },
+          {
+            "path": "apps/web/components/settings/EnableTwoFactorModal.tsx",
+            "line": 64,
+            "body": "**CODE_MAINTAINABILTIY**: The `backupCodes` state is typed as an empty `any[]` array via `useState([])` instead of `useState<string[]>([])`. This removes TypeScript safety on all downstream usages: `backupCodes.map(formatBackupCode)`, `backupCodes.indexOf(...)`, and `navigator.clipboard.writeText(backupCodes.map(formatBackupCode).join(...))`. If the API response body changes shape or contains unexpected values, TypeScript will not catch the mismatch. \n Using `useState([])` infers type `never[]`, but TypeScript is permissive enough here to treat it as `any[]` in some configurations. All downstream uses of `backupCodes` lose type safety, preventing compiler detection of mismatched API response shapes or broken `formatBackupCode` calls.\n```\nconst [backupCodes, setBackupCodes] = useState<string[]>([]);\n``` ",
+            "created_at": "2026-08-19T23:18:34Z"
+          },
+          {
+            "path": "apps/web/components/settings/EnableTwoFactorModal.tsx",
+            "line": 71,
+            "body": "**CODE_MAINTAINABILTIY**: The `resetState` function in `EnableTwoFactorModal` resets `password`, `errorMessage`, and `step`, but does NOT reset `backupCodes` or `backupCodesUrl`. This means that after closing the backup codes dialog (via the 'Close' button calling `resetState()`) and re-opening the modal without completing a new setup, the stale backup codes from the previous session remain in state and can be displayed. The object URL created with `URL.createObjectURL` is also not revoked on modal close, which is a resource leak. \n After a user completes 2FA setup and views backup codes, clicking 'Close' calls `resetState()` then `onEnable()`. If the parent component keeps the modal mounted and reopens it (e.g., re-triggering an enable flow), the stale `backupCodes` state will persist and be shown in `DisplayBackupCodes` step if navigated to directly. Additionally, `URL.createObjectURL` allocates a browser resource that is only revoked on setup re-entry (`if (backupCodesUrl) URL.revokeObjectURL(...)`), but never revoked on close without a new setup — leaking the object URL.\n```\nconst resetState = () => {\n  setPassword(\"\");\n  setErrorMessage(null);\n  setStep(SetupStep.ConfirmPassword);\n  setBackupCodes([]);\n  if (backupCodesUrl) {\n    URL.revokeObjectURL(backupCodesUrl);\n    setBackupCodesUrl(\"\");\n  }\n};\n``` ",
+            "created_at": "2026-08-19T23:18:35Z"
+          },
+          {
+            "path": "apps/web/components/settings/EnableTwoFactorModal.tsx",
+            "line": 276,
+            "body": "**CODE_MAINTAINABILTIY**: The `navigator.clipboard.writeText()` call on the 'Copy' button does not handle the returned Promise. If the browser denies clipboard access (e.g., clipboard permissions are blocked, or the document is not focused), the call will silently fail, but the success toast `showToast(t(\"backup_codes_copied\"), \"success\")` is still unconditionally shown to the user, misleading them into believing their codes were copied. \n `navigator.clipboard.writeText` returns a Promise that resolves on success or rejects on permission denial or API unavailability. The current code ignores the rejection path and always shows the success toast, so users are told their backup codes were copied even when they were not. This is a concrete failure mode where clipboard access is denied (common on HTTP origins, in iframes, or when clipboard permissions are blocked).\n```\nonClick={async (e) => {\n  e.preventDefault();\n  try {\n    await navigator.clipboard.writeText(backupCodes.map(formatBackupCode).join(\"\n\"));\n    showToast(t(\"backup_codes_copied\"), \"success\");\n  } catch {\n    showToast(t(\"something_went_wrong\"), \"error\");\n  }\n}}\n``` ",
+            "created_at": "2026-08-19T23:18:36Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "**ERROR**: The `apps/web/components/security/DisableTwoFactorModal.tsx` file contains a separate (unmigrated) copy of the `TwoFactorAuthAPI.disable` caller that still uses the old 2-argument signature: `TwoFactorAuthAPI.disable(password, totpCode)`. This is a different file from `apps/web/components/settings/DisableTwoFactorModal.tsx` which was updated in this PR. The security modal's copy of `TwoFactorAuthAPI` (`apps/web/components/security/TwoFactorAuthAPI.ts`) was not updated to include the `backupCode` parameter. Users accessing the 2FA disable flow through the security settings page (as opposed to the settings page) will be unable to use backup codes. \n The PR updates `apps/web/components/settings/DisableTwoFactorModal.tsx` and `apps/web/components/settings/TwoFactorAuthAPI.ts` to support backup codes, but there is an identical parallel `DisableTwoFactorModal.tsx` and `TwoFactorAuthAPI.ts` in `apps/web/components/security/` that was not updated. The security path's `disable` call still uses the old 2-argument form, meaning it won't send `backupCode` to the API — users who access 2FA settings via the security page cannot use backup codes to disable 2FA.\n```\n// In apps/web/components/security/TwoFactorAuthAPI.ts, update disable signature:\nasync disable(password: string, code: string, backupCode: string) {\n  return fetch(\"/api/auth/two-factor/totp/disable\", {\n    method: \"POST\",\n    body: JSON.stringify({ password, code, backupCode }),\n    headers: {\n      \"Content-Type\": \"application/json\",\n    },\n  });\n},\n\n// Also update apps/web/components/security/DisableTwoFactorModal.tsx\n// to add backup code state and UI (similar to the settings modal).\n``` ",
+            "created_at": "2026-08-19T23:18:30Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 54486a059cd2032042189bb565646ba4e0f6bd61.",
+            "created_at": "2026-08-19T23:18:38Z"
+          }
+        ]
       }
     ]
   },
@@ -74331,6 +76185,61 @@
             "created_at": "2026-06-28T22:35:43Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR10967__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR10967__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "packages/app-store/googlecalendar/lib/CalendarService.ts",
+            "line": 256,
+            "body": "**SECURITY**: The fallback reuses `externalCalendarId` as the search key only after the ternary has established that it is falsy. It therefore cannot select the intended destination calendar: `updateEvent` passes the resulting `undefined` calendar ID to Google, while the identical delete path falls back to `primary` instead of the event's destination. Fall back to the appropriate destination entry, such as the first/main-host calendar, rather than searching by the absent ID. \n Updates without an external calendar ID fail to select a calendar, and deletes can be attempted against `primary` instead of the stored destination.\n```\nconst selectedCalendar = externalCalendarId\n  ? externalCalendarId\n  : event.destinationCalendar?.[0]?.externalId ?? \"primary\";\n\nconst calendarId = externalCalendarId\n  ? externalCalendarId\n  : event.destinationCalendar?.[0]?.externalId ?? defaultCalendarId;\n``` ",
+            "created_at": "2026-08-19T23:23:45Z"
+          },
+          {
+            "path": "packages/core/EventManager.ts",
+            "line": 542,
+            "body": "**CODE_MAINTAINABILTIY**: `credential` can remain undefined when it is absent from `this.calendarCredentials` and the database lookup returns no usable credential, but `updateEvent` is still called unconditionally. Guard this call as the creation path does, and handle or skip references whose credential cannot be resolved. \n A calendar reference with an unresolved credential is passed to `updateEvent`, causing the update path to fail when it accesses credential data.\n```\nif (credential) {\n  result.push(updateEvent(credential, event, bookingRefUid, calenderExternalId));\n} else {\n  log.error(`Could not find credential for reference ${reference.credentialId}, skipping update`);\n}\n``` ",
+            "created_at": "2026-08-19T23:23:46Z"
+          },
+          {
+            "path": "packages/core/EventManager.ts",
+            "line": 119,
+            "body": "**CODE_MAINTAINABILTIY**: `mainHostDestinationCalendar` is undefined when `evt.destinationCalendar` is null, undefined, or empty, so evaluating `.integration` crashes Google Meet booking creation before the Daily fallback can run. Use optional chaining on the destructured value. \n A Meet booking without a destination calendar dereferences `undefined` during event creation.\n```\nconst [mainHostDestinationCalendar] = evt.destinationCalendar ?? [];\nif (evt.location === MeetLocationType && mainHostDestinationCalendar?.integration !== \"google_calendar\") {\n  evt[\"location\"] = \"integrations:daily\";\n}\n``` ",
+            "created_at": "2026-08-19T23:23:47Z"
+          },
+          {
+            "path": "packages/core/EventManager.ts",
+            "line": 344,
+            "body": "**PERFORMANCE_OPTIMIZATION**: This `findUnique` is awaited inside the loop over destination calendars. A collective event with multiple credentials missing from `this.calendarCredentials` therefore performs one database round trip at a time. Collect the missing credential IDs and fetch them with one `findMany` before entering the loop. \n Creating an event for N uncached host credentials incurs N sequential database queries.\n```\nconst missingCredentialIds = event.destinationCalendar\n  .filter((destination) => destination.credentialId && !this.calendarCredentials.find((c) => c.id === destination.credentialId))\n  .map((destination) => destination.credentialId as number);\n\nconst credentialsFromDB = await prisma.credential.findMany({\n  include: { app: { select: { slug: true } } },\n  where: { id: { in: missingCredentialIds } },\n});\n``` ",
+            "created_at": "2026-08-19T23:23:47Z"
+          },
+          {
+            "path": "packages/core/EventManager.ts",
+            "line": 517,
+            "body": "**PERFORMANCE_OPTIMIZATION**: This `findUnique` is awaited for each calendar reference whose credential is not already loaded, serializing all database lookups during a collective reschedule. Batch-fetch the missing credential IDs before the loop and resolve each reference from a lookup map. \n Updating N references with uncached credentials incurs N sequential database round trips.\n```\nconst missingRefCredentialIds = calendarReference\n  .filter((reference) => reference.credentialId && !this.calendarCredentials.find((c) => c.id === reference.credentialId))\n  .map((reference) => reference.credentialId as number);\n\nconst batchedCredentials = await prisma.credential.findMany({\n  include: { app: { select: { slug: true } } },\n  where: { id: { in: missingRefCredentialIds } },\n});\nconst batchedCredentialMap = new Map(batchedCredentials.map((credential) => [credential.id, credential]));\n``` ",
+            "created_at": "2026-08-19T23:23:48Z"
+          },
+          {
+            "path": "packages/features/bookings/lib/handleCancelBooking.ts",
+            "line": 433,
+            "body": "**PERFORMANCE_OPTIMIZATION**: This lookup is awaited inside the loop over calendar references, so cancelling a collective booking performs one serial database query for every credential missing from `bookingToDelete.user.credentials`. Batch-fetch those credential IDs before the loop and use an ID-indexed map. \n Cancellation latency grows by one sequential database round trip per unresolved calendar reference.\n```\nconst missingCancelCredentialIds = bookingCalendarReference\n  .filter((reference) => reference.credentialId && !bookingToDelete.user.credentials.find((c) => c.id === reference.credentialId))\n  .map((reference) => reference.credentialId as number);\n\nconst fetchedCancelCredentials = await prisma.credential.findMany({\n  where: { id: { in: missingCancelCredentialIds } },\n});\nconst cancelCredentialMap = new Map(fetchedCancelCredentials.map((credential) => [credential.id, credential]));\n``` ",
+            "created_at": "2026-08-19T23:23:49Z"
+          },
+          {
+            "path": "packages/features/bookings/lib/handleNewBooking.ts",
+            "line": 1078,
+            "body": "**ERROR**: When neither the event type nor organizer has a destination calendar, `evt.destinationCalendar` is initialized to `null`. Optional chaining then skips this `push`, dropping every collected team destination calendar for that collective event. Initialize the array before appending. \n Collective events without an organizer destination calendar silently omit all team-host destination calendars.\n```\nif (isTeamEventType && eventType.schedulingType === \"COLLECTIVE\") {\n  if (!evt.destinationCalendar) {\n    evt.destinationCalendar = [];\n  }\n  evt.destinationCalendar.push(...teamDestinationCalendars);\n}\n``` ",
+            "created_at": "2026-08-19T23:23:50Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit de628295646d0848226618108a52f2f1e5d04ac0.",
+            "created_at": "2026-08-19T23:23:51Z"
+          }
+        ]
       }
     ]
   },
@@ -75831,6 +77740,49 @@
             "created_at": "2026-06-28T22:13:21Z"
           }
         ]
+      },
+      {
+        "tool": "deputydev-v1",
+        "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR8087__20260819",
+        "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR8087__20260819/pull/1",
+        "review_comments": [
+          {
+            "path": "packages/app-store/vital/lib/reschedule.ts",
+            "line": 125,
+            "body": "**SECURITY**: `forEach` discards the promises returned by this async callback. Consequently, `Reschedule` can finish before the calendar/video deletions settle, and the surrounding `try/catch` cannot handle callback rejections. Await the deletion work with `Promise.all(...map(...))` or an awaited `for...of` loop. \n Cleanup runs outside the function's awaited chain, so success can be returned before deletion completes and failures can become unhandled rejections.\n```\ntry {\n  await Promise.all(\n    bookingRefsFiltered.map(async (bookingRef) => {\n      if (bookingRef.uid) {\n        if (bookingRef.type.endsWith(\"_calendar\")) {\n          const calendar = await getCalendar(credentialsMap.get(bookingRef.type));\n          return calendar?.deleteEvent(bookingRef.uid, builder.calendarEvent);\n        } else if (bookingRef.type.endsWith(\"_video\")) {\n          return deleteMeeting(credentialsMap.get(bookingRef.type), bookingRef.uid);\n        }\n      }\n    })\n  );\n} catch (error) {\n  if (error instanceof Error) {\n    logger.error(error.message);\n  }\n}\n``` ",
+            "created_at": "2026-08-19T23:12:35Z"
+          },
+          {
+            "path": "packages/app-store/wipemycalother/lib/reschedule.ts",
+            "line": 125,
+            "body": "**SECURITY**: `forEach` does not await this async callback, so calendar/video deletion continues after `Reschedule` proceeds and rejected callbacks bypass the surrounding `try/catch`. Collect and await the callbacks with `Promise.all(...map(...))` or use an awaited `for...of` loop. \n The deletion promises are discarded, allowing the function to complete before cleanup and leaving callback failures unobserved.\n```\ntry {\n  await Promise.all(\n    bookingRefsFiltered.map(async (bookingRef) => {\n      if (bookingRef.uid) {\n        if (bookingRef.type.endsWith(\"_calendar\")) {\n          const calendar = await getCalendar(credentialsMap.get(bookingRef.type));\n          return calendar?.deleteEvent(bookingRef.uid, builder.calendarEvent);\n        } else if (bookingRef.type.endsWith(\"_video\")) {\n          return deleteMeeting(credentialsMap.get(bookingRef.type), bookingRef.uid);\n        }\n      }\n    })\n  );\n} catch (error) {\n  if (error instanceof Error) {\n    logger.error(error.message);\n  }\n}\n``` ",
+            "created_at": "2026-08-19T23:12:36Z"
+          },
+          {
+            "path": "packages/core/videoClient.ts",
+            "line": 24,
+            "body": "**PERFORMANCE_OPTIMIZATION**: This `await` inside the `for...of` serializes the independent app module resolutions: the next credential is not processed until the previous `appStore` promise resolves. Resolve the entries concurrently with `Promise.all` and then build the adapters in credential order. \n Users with multiple video credentials incur the sum of the module-resolution delays instead of allowing those independent resolutions to overlap.\n```\nconst getVideoAdapters = async (withCredentials: CredentialPayload[]): Promise<VideoApiAdapter[]> => {\n  const apps = await Promise.all(\n    withCredentials.map((cred) => {\n      const appName = cred.type.split(\"_\").join(\"\");\n      return appStore[appName as keyof typeof appStore];\n    })\n  );\n\n  return apps.reduce<VideoApiAdapter[]>((videoAdapters, app, i) => {\n    if (app && \"lib\" in app && \"VideoApiAdapter\" in app.lib) {\n      const makeVideoApiAdapter = app.lib.VideoApiAdapter as VideoApiAdapterFactory;\n      videoAdapters.push(makeVideoApiAdapter(withCredentials[i]));\n    }\n    return videoAdapters;\n  }, []);\n};\n``` ",
+            "created_at": "2026-08-19T23:12:37Z"
+          },
+          {
+            "path": "packages/features/bookings/lib/handleCancelBooking.ts",
+            "line": 460,
+            "body": "**CODE_MAINTAINABILTIY**: This async `forEach` is not awaited. Each callback pauses at `getCalendar` before deleting events and mutating `apiDeletes`, so downstream `Promise.all(apiDeletes)` can run before these callbacks finish and therefore does not wait for this recurring cleanup. Use an awaited `for...of` loop or await a `Promise.all` over `map`. \n The recurring calendar deletions are detached from the handler's awaited flow, so cancellation can proceed without waiting for them and their failures are not observed there.\n```\nconst calendarCreds = bookingToDelete.user.credentials.filter((credential) =>\n  credential.type.endsWith(\"_calendar\")\n);\nfor (const credential of calendarCreds) {\n  const calendar = await getCalendar(credential);\n  for (const updBooking of updatedBookings) {\n    const bookingRef = updBooking.references.find((ref) => ref.type.includes(\"_calendar\"));\n    if (bookingRef) {\n      const { uid, externalCalendarId } = bookingRef;\n      apiDeletes.push(calendar?.deleteEvent(uid, evt, externalCalendarId) as Promise<unknown>);\n    }\n  }\n}\n``` ",
+            "created_at": "2026-08-19T23:12:38Z"
+          },
+          {
+            "path": "packages/trpc/server/routers/viewer/bookings.tsx",
+            "line": 553,
+            "body": "**CODE_MAINTAINABILTIY**: `forEach` discards these async callback promises. The handler therefore proceeds to send the reschedule email and return without waiting for calendar/video deletion, while deletion failures are not propagated through the mutation. Await `Promise.all(bookingRefsFiltered.map(...))` or use an awaited `for...of` loop. \n The mutation can report rescheduling success before old integration events are removed, and rejected deletion promises are left unobserved.\n```\nawait Promise.all(\n  bookingRefsFiltered.map(async (bookingRef) => {\n    if (bookingRef.uid) {\n      if (bookingRef.type.endsWith(\"_calendar\")) {\n        const calendar = await getCalendar(credentialsMap.get(bookingRef.type));\n        return calendar?.deleteEvent(\n          bookingRef.uid,\n          builder.calendarEvent,\n          bookingRef.externalCalendarId\n        );\n      } else if (bookingRef.type.endsWith(\"_video\")) {\n        return deleteMeeting(credentialsMap.get(bookingRef.type), bookingRef.uid);\n      }\n    }\n  })\n);\n``` ",
+            "created_at": "2026-08-19T23:12:39Z"
+          },
+          {
+            "path": null,
+            "line": null,
+            "body": "DeputyDev has completed a review of your pull request for commit 820d7fa87e0c824c5dee8082a33efadb1a2566f7.",
+            "created_at": "2026-08-19T23:12:40Z"
+          }
+        ]
       }
     ]
   }
diff --git a/offline/results/gpt-5.5/candidates.json b/offline/results/gpt-5.5/candidates.json
new file mode 100644
index 0000000..6e07456
--- /dev/null
+++ b/offline/results/gpt-5.5/candidates.json
@@ -0,0 +1,2217 @@
+{
+  "https://github.com/keycloak/keycloak/pull/37429": {
+    "deputydev-v1": [
+      {
+        "text": "VerifyMessageProperties initializes immutable HTML policy factories and recompiles the reusable HTML_TAGS regex for every instance even though they do not depend on instance state, so creating a verifier per properties file repeatedly rebuilds the same policies and pattern instead of declaring them private static final constants.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java",
+        "source_comment_line": 58
+      },
+      {
+        "text": "The anchor-mismatch diagnostic does not include the property key or file, so builds processing many resource bundles cannot identify which translated property produced the missing-anchor message, unlike the nearby illegal-HTML diagnostic.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java",
+        "source_comment_line": 157
+      },
+      {
+        "text": "messages_lt.properties assigns the Italian sentence \"Installa una delle seguenti applicazioni sul tuo cellulare:\" to loginTotpStep1, so Lithuanian users would see Italian text during TOTP setup instead of an appropriate Lithuanian translation.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "themes/src/main/resources-community/theme/base/login/messages/messages_lt.properties",
+        "source_comment_line": 71
+      }
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/37634": {
+    "deputydev-v1": [
+      {
+        "text": "The grant shortcut documentation must specify an exactly two-character shortcut rather than a three-letter shortcut; the token-context decoder expects a six-character prefix composed of three two-character segments, so following the current Javadoc would produce an incompatible grant shortcut.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "server-spi-private/src/main/java/org/keycloak/protocol/oidc/grants/OAuth2GrantTypeFactory.java",
+        "source_comment_line": 30
+      },
+      {
+        "text": "The constructor repeats the null check for `grantType` while the error message refers to `rawTokenId`, so `rawTokenId` is stored without validation and an `AccessTokenContext` can be constructed with that required field unset.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "services/src/main/java/org/keycloak/protocol/oidc/encode/AccessTokenContext.java",
+        "source_comment_line": 73
+      },
+      {
+        "text": "The lazy cache lookup in `getShortcutByGrantType` uses a non-atomic get/null-check/discovery/put sequence, so concurrent misses for the same grant can repeat `getProviderFactory` calls before either caller populates the `ConcurrentHashMap`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "services/src/main/java/org/keycloak/protocol/oidc/encode/DefaultTokenContextEncoderProviderFactory.java",
+        "source_comment_line": 108
+      },
+      {
+        "text": "The lazy cache lookup in `getGrantTypeByShortcut` uses a non-atomic get/null-check/discovery/put sequence, so concurrent misses for the same shortcut can repeat the full provider stream scan before either caller populates the `ConcurrentHashMap`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "services/src/main/java/org/keycloak/protocol/oidc/encode/DefaultTokenContextEncoderProviderFactory.java",
+        "source_comment_line": 108
+      },
+      {
+        "text": "The newly introduced `TokenContextEncoderProvider` contract methods lack method-level Javadoc defining `rawTokenId` versus `encodedTokenId`, the returned context, and relevant failure behavior, leaving callers and implementations without a specified encoding contract.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "services/src/main/java/org/keycloak/protocol/oidc/encode/TokenContextEncoderProvider.java",
+        "source_comment_line": 33
+      },
+      {
+        "text": "The event matcher extracts the grant shortcut from the wrong substring and uses the equality check with the wrong polarity; it should compare `items[0].substring(4, 6)` against `expectedGrantShortcut` and return false when they differ, otherwise it reads across the token-type/grant-type boundary and rejects matching shortcuts.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AssertEvents.java",
+        "source_comment_line": 483
+      }
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/38446": {
+    "deputydev-v1": [
+      {
+        "text": "`getCredentials` dereferences the result of `getMyUser(user)` without checking for null; if the user is absent from the internal users map, `getMyUser(user)` returns null and accessing `myUser.recoveryCodes` throws a `NullPointerException` instead of returning an empty credential stream.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java",
+        "source_comment_line": 232
+      },
+      {
+        "text": "Successful federated recovery-code authentication only checks that the submitted code is present in the stored list and does not remove or persist removal of the matched code, leaving the same recovery code valid for indefinite replay instead of enforcing one-time use.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java",
+        "source_comment_line": 340
+      }
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/36882": {
+    "deputydev-v1": [
+      {
+        "text": "The feature-disabled test for the update metadata invocation asserts only the error text and does not assert exit code 4, so a regression returning a different exit code would still pass even though automation relies on the documented feature-disabled exit-code contract.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/UpdateCommandDistTest.java",
+        "source_comment_line": 49
+      },
+      {
+        "text": "There is no disabled-feature test coverage for the UpdateCompatibilityCheck sub-command; only the metadata sub-command is launched, so removing or breaking the equivalent feature gate in the check sub-command would not be detected.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/UpdateCommandDistTest.java",
+        "source_comment_line": 47
+      }
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/36880": {
+    "deputydev-v1": [
+      {
+        "text": "When V2 admin permissions are active, the client-removal listener can call `clients().setPermissionsEnabled(client, false)`, which routes through `management(...)` to `MgmtPermissionsV2` and then `ClientPermissionsV2.setPermissionsEnabled`, a method that unconditionally throws `UnsupportedOperationException`; as a result, deleting a client in V2 fails during event processing unless the cleanup is restricted to V1 or implemented with a V2-compatible deletion path.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/AdminPermissions.java",
+        "source_comment_line": 91
+      },
+      {
+        "text": "The Javadocs for `requireView()` and `requireViewClientScopes()` incorrectly describe boolean return values even though both methods return `void`; their contracts should instead state that they throw `ForbiddenException` when `canView()` or `canViewClientScopes()` returns `false`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java",
+        "source_comment_line": 98
+      },
+      {
+        "text": "Existing client management-permissions REST endpoints can select the V2 implementation, but `ClientPermissionsV2.isPermissionsEnabled`, `setPermissionsEnabled`, `resource`, and `getPermissions` all unconditionally throw `UnsupportedOperationException`, so GET or update requests to those public endpoints produce unhandled 500 errors in V2 unless the V1 endpoints are prevented from routing to this implementation or meaningful V2 behavior is provided.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java",
+        "source_comment_line": 193
+      },
+      {
+        "text": "The private `getEvaluationContext(ClientModel, AccessToken)` method in `ClientPermissionsV2` has no callers while the actual `hasPermission` paths call `root.evaluatePermission(...)` directly, making the method dead code that misleadingly suggests a client-specific evaluation context is used.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java",
+        "source_comment_line": 262
+      },
+      {
+        "text": "`ClientPermissionsV2` contains an unused static import of `AdminPermissionManagement.TOKEN_EXCHANGE`; the constant is never referenced in the class and the import should be removed.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java",
+        "source_comment_line": 46
+      },
+      {
+        "text": "Both `hasPermission` overloads in `ClientPermissionsV2` allocate `Arrays.asList(scope)` only to test membership against a single scope value, causing unnecessary allocation on every invocation; they should compare directly with `scope.equals(s)` as `hasGrantedPermission` already does.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java",
+        "source_comment_line": 225
+      }
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/37038": {
+    "deputydev-v1": [
+      {
+        "text": "The Javadoc for `requireManageMembers(GroupModel)` is copy-pasted from `requireManageMembership` and incorrectly references `canManageMembership(GroupModel)` instead of the semantically correct `canManageMembers(GroupModel)`, misleading API consumers and implementors about which permission check the method enforces.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionEvaluator.java",
+        "source_comment_line": 133
+      },
+      {
+        "text": "The Javadoc for `getGroupIdsWithViewPermission()` describes the return value as a stream even though the method signature returns `Set<String>`, giving callers an incorrect API contract instead of documenting a set of group IDs.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionEvaluator.java",
+        "source_comment_line": 142
+      },
+      {
+        "text": "`GroupPermissionsV2.canManage()` passes both `VIEW` and `MANAGE` scopes to `hasPermission`, whose loop succeeds if any requested scope is granted, so a caller with only all-groups `VIEW` permission can pass a manage authorization gate; the check should require `AdminPermissionsSchema.MANAGE` only.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java",
+        "source_comment_line": 70
+      },
+      {
+        "text": "The callback passes `groupResource.getId()` to `hasPermission`, but that value is the authorization resource UUID rather than the group model ID stored in `groupResource.getName()`, causing `findByName` to miss the group-specific permission resource and fall back to all-groups evaluation.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java",
+        "source_comment_line": 122
+      },
+      {
+        "text": "The callback adds `groupResource.getId()` to the returned `granted` set, but that value is the authorization resource UUID rather than the group model ID stored in `groupResource.getName()`, so provider-side user filtering receives invalid group IDs.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java",
+        "source_comment_line": 122
+      }
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/33832": {
+    "deputydev-v1": [
+      {
+        "text": "The added Hamcrest test dependency in authz/client/pom.xml is unused because the module's tests use org.junit.Assert.assertArrayEquals and have no Hamcrest imports or matcher usage, so it adds unnecessary build configuration.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "authz/client/pom.xml",
+        "source_comment_line": 68
+      },
+      {
+        "text": "ASN1Decoder.readLength() returns -1 for indefinite-length encodings, but the code enters a while (length > 0) path that skips parsing and returns an empty sequence instead of rejecting the unsupported DER encoding, causing malformed indefinite-length input to fail later with a misleading sequence-size error.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/ASN1Decoder.java",
+        "source_comment_line": 55
+      },
+      {
+        "text": "AuthzClientCryptoProvider.getBouncyCastleProvider() returns the provider backing the JDK default keystore type instead of an actual BouncyCastle provider or an explicit unsupported signal, so callers expecting BouncyCastle can receive an unrelated JDK keystore provider and fail on provider-specific algorithm lookups.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/AuthzClientCryptoProvider.java",
+        "source_comment_line": 61
+      },
+      {
+        "text": "The two standalone ASN1Encoder.create().write(...) calls in AuthzClientCryptoProvider discard their results and are immediately repeated in the return expression, so they have no effect on the output and cause redundant allocations and DER encoding work on every invocation.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/AuthzClientCryptoProvider.java",
+        "source_comment_line": 114
+      }
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/40940": {
+    "deputydev-v1": [
+      {
+        "text": "GroupAdapter.getSubGroupsCount() returns null when modelSupplier.get() is absent, violating the GroupModel.getSubGroupsCount() contract that it never returns null and allowing null propagation to representations or unboxing failures in callers; it should return a non-null fallback such as 0L.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/GroupAdapter.java",
+        "source_comment_line": 275
+      },
+      {
+        "text": "The adjacent getSubGroupsStream(...) overloads still directly dereference modelSupplier.get(), so if the supplied model is null during concurrent deletion they can throw an NPE; they should guard the null model case and return an empty stream.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/GroupAdapter.java",
+        "source_comment_line": 274
+      },
+      {
+        "text": "The test checks caughtExceptions without retaining and joining the reader thread, so the reader may still be inside groups() and append an exception after the assertion has already passed, causing false passes.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java",
+        "source_comment_line": 155
+      },
+      {
+        "text": "The test does not synchronize or record that at least one reader call completed during the deletion loop, so the deletions can finish without actually exercising the concurrent-read regression path.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java",
+        "source_comment_line": 155
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1": {
+    "deputydev-v1": [
+      {
+        "text": "In `UsernameForm.java`, the newly added zero-argument call to `isConditionalPasskeysEnabled()` does not compile because the available inherited method requires a `UserModel` argument; pass the already-checked user, e.g. `isConditionalPasskeysEnabled(context.getUser())`, when `context.getUser() != null`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "services/src/main/java/org/keycloak/authentication/authenticators/browser/UsernameForm.java",
+        "source_comment_line": 47
+      }
+    ]
+  },
+  "https://github.com/getsentry/sentry/pull/93824": {
+    "deputydev-v1": [
+      {
+        "text": "The restart limit guard uses `>` before incrementing `self.process_restarts[process_index]`, so a worker whose restart count already equals `MAX_PROCESS_RESTARTS` is restarted one extra time and the RuntimeError is only raised after the following failure; the check should use `>= MAX_PROCESS_RESTARTS`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "src/sentry/spans/consumers/process/flusher.py",
+        "source_comment_line": 247
+      },
+      {
+        "text": "The `SpanFlusher` docstring incorrectly says it creates one process per shard, but `num_processes` is capped by `max_processes` and the round-robin mapping can assign multiple shards to each worker, so the documentation contradicts the implemented capped shared-shard worker model.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "src/sentry/spans/consumers/process/flusher.py",
+        "source_comment_line": 31
+      },
+      {
+        "text": "The `spans.buffer.flusher.wait_produce` timer uses the tag key `shards` while neighboring metrics use `shard` for the same `shard_tag` value, creating an inconsistent metric dimension that causes queries and dashboards filtering on `shard` to omit this timer.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "src/sentry/spans/consumers/process/flusher.py",
+        "source_comment_line": 199
+      },
+      {
+        "text": "Negative `max_processes` values are accepted and remain truthy, causing `num_processes` to become negative and leaving no valid process entries for shard assignment; the constructor should reject `max_processes < 1` before building the worker-to-shard mapping.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "src/sentry/spans/consumers/process/flusher.py",
+        "source_comment_line": 51
+      },
+      {
+        "text": "When `produce_to_pipe` is set, the worker is a `threading.Thread`, but the restart cleanup only handles multiprocessing workers before starting a replacement, so a hung thread can remain alive while another worker is started for the same shards across repeated restarts.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "src/sentry/spans/consumers/process/flusher.py",
+        "source_comment_line": 254
+      },
+      {
+        "text": "The comment describes an individual-shard health check, but the code reads a single `process_healthy_since[process_index]` timestamp and marks the entire worker as hung, so the comment should describe a process-level hang check instead of shard-specific health evaluation.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "src/sentry/spans/consumers/process/flusher.py",
+        "source_comment_line": 235
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5": {
+    "deputydev-v1": [
+      {
+        "text": "The age/timestamp mutual-exclusivity validation uses truthiness checks instead of presence checks, so zero-valued fields such as age=0 with timestamp or timestamp=0 with age can bypass validation and allow both mutually exclusive fields in the same payload.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "src/sentry/issues/endpoints/browser_reporting_collector.py",
+        "source_comment_line": 52
+      },
+      {
+        "text": "The code pairs error_ids with events.values() by position even though nodestore get_multi may return a dictionary in a different order or omit entries, which can attach an event's title, message, and timestamp to the wrong error ID; returned events should be associated by node_id using a node_id-to-event_id mapping.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py",
+        "source_comment_line": 118
+      },
+      {
+        "text": "When the use-table-widget-visualization feature is enabled, the code passes empty columns, data, fields, and units to TableWidgetVisualization instead of using result.data and result.meta, causing organizations with the feature enabled to render a blank table regardless of available query results.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "static/app/views/dashboards/widgetCard/chart.tsx",
+        "source_comment_line": 164
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1": {
+    "deputydev-v1": [
+      {
+        "text": "The audit-log endpoint has two branches that duplicate the same `self.paginate()` arguments and differ only in `paginator_cls` and `enable_advanced_features`, creating two locations where shared request, queryset, ordering, and serialization arguments can diverge during future changes.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "src/sentry/api/endpoints/organization_auditlogs.py",
+        "source_comment_line": 73
+      },
+      {
+        "text": "`OptimizedCursorPaginator.get_item_key` passes the ordering value directly to `math.floor` or `math.ceil`, but the audit-log endpoint configures it with `order_by=\"-datetime\"`, so the value is a `datetime` and optimized cursor generation raises `TypeError` on every response unless datetime keys are converted to the numeric representation used by the existing datetime paginator.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "src/sentry/api/paginator.py",
+        "source_comment_line": 840
+      },
+      {
+        "text": "The optimized paginator branch for `cursor.offset < 0` assigns the negative offset to `start_offset` and uses it as the start of a Django QuerySet slice, but Django rejects negative QuerySet indices, causing optimized requests with such cursors to raise an unhandled exception instead of returning a page.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "src/sentry/api/paginator.py",
+        "source_comment_line": 882
+      },
+      {
+        "text": "The paginator comment incorrectly states that negative offsets are safe because the underlying queryset handles boundary conditions, but Django QuerySets reject negative slice indices, so the documentation should remove that safety claim or document the required non-negative offset invariant.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "src/sentry/api/paginator.py",
+        "source_comment_line": 179
+      },
+      {
+        "text": "`OptimizedCursorPaginator.get_result` duplicates the base paginator's cursor initialization, hit counting, boundary trimming, reversal, cursor construction, and post-query filtering even though only offset selection differs, so fixes or cursor-handling changes made to the base pagination flow may not reach the optimized paginator and the implementations can diverge.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "src/sentry/api/paginator.py",
+        "source_comment_line": 821
+      }
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/97529": {
+    "deputydev-v1": [
+      {
+        "text": "In `NewResourceServer`, logging the error returned by `s.Init(ctx)` is redundant because `server.Init` already logs `s.log.Error(\"error initializing resource server\", \"error\", s.initErr)` before returning the same failure, causing two identical \"error initializing resource server\" log lines for a single initialization failure.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "pkg/storage/unified/resource/server.go",
+        "source_comment_line": 258
+      }
+    ]
+  },
+  "https://github.com/getsentry/sentry/pull/80168": {
+    "deputydev-v1": [
+      {
+        "text": "`MetricAlertDetectorHandler` now extends `StatefulDetectorHandler` without implementing its abstract hooks, including `build_occurrence_and_event_data`, so Python treats the handler as abstract and raises `TypeError` when it is instantiated, blocking any path that resolves this detector handler.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "src/sentry/incidents/grouptype.py",
+        "source_comment_line": 12
+      },
+      {
+        "text": "The `evaluate` docstring still says the method returns a list even though its annotation and implementation now return a `dict[DetectorGroupKey, DetectorEvaluationResult]`, causing the documentation to directly contradict the code.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "src/sentry/workflow_engine/processors/detector.py",
+        "source_comment_line": 226
+      }
+    ]
+  },
+  "https://github.com/getsentry/sentry/pull/80528": {
+    "deputydev-v1": [
+      {
+        "text": "`create_incident_occurrence` is called for every check-in in `incidents.py`, causing `monitor_env.get_last_successful_checkin()` to be fetched repeatedly for the same monitor environment; incidents containing multiple check-ins perform the same last-successful-check-in lookup once per occurrence instead of once per monitor environment, so the value should be fetched once before the loop and passed in.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "src/sentry/monitors/logic/incident_occurrence.py",
+        "source_comment_line": 46
+      },
+      {
+        "text": "The function copies the monitor config and replaces `schedule_type` with its display value, but returns `monitor_environment.monitor.config` instead of the local transformed `config`, discarding the transformation and exposing the original `schedule_type` value in the returned monitor context.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "src/sentry/monitors/logic/incident_occurrence.py",
+        "source_comment_line": 168
+      },
+      {
+        "text": "`get_human_failure_reason` has no branch for a zero total, so when the input is empty or every status is excluded by `HUMAN_FAILURE_STATUS_MAP`, `get_text_list([])` produces an empty value and the function returns the malformed user-facing string `\" check-ins detected\"` instead of the documented fallback.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "src/sentry/monitors/logic/incident_occurrence.py",
+        "source_comment_line": 145
+      }
+    ]
+  },
+  "https://github.com/getsentry/sentry/pull/77754": {
+    "deputydev-v1": [
+      {
+        "text": "`timezone.now()` is evaluated once when `AssignmentSource` is defined, so every instance that omits `queued` receives the same stale module-import timestamp instead of a fresh timestamp; use `field(default_factory=timezone.now)` for the `queued` dataclass field.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "src/sentry/integrations/services/assignment_source.py",
+        "source_comment_line": 18
+      },
+      {
+        "text": "The test name incorrectly describes its input as an array even though it passes an empty dictionary; rename it to `test_from_dict_empty_dict` so the test name accurately reflects the tested input.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "tests/sentry/integrations/services/test_assignment_source.py",
+        "source_comment_line": 8
+      },
+      {
+        "text": "The test name `test_from_dict_inalid_data` contains a typo; rename it to `test_from_dict_invalid_data` so the identifier is readable and appears correctly in test reports.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "tests/sentry/integrations/services/test_assignment_source.py",
+        "source_comment_line": 13
+      }
+    ]
+  },
+  "https://github.com/getsentry/sentry/pull/95633": {
+    "deputydev-v1": [
+      {
+        "text": "The worker unconditionally marks an offset complete in the finally block even when result_processor raises and the exception is swallowed, making the failed message eligible for commit and preventing redelivery despite the strategy's guarantee that offsets are committed only after successful processing.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py",
+        "source_comment_line": 148
+      },
+      {
+        "text": "The work queue is unbounded, so if producers submit messages faster than workers process them, queued WorkItems retain full messages and can cause unbounded heap growth instead of applying backpressure.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py",
+        "source_comment_line": 185
+      },
+      {
+        "text": "shutdown sets each worker's shutdown flag before gracefully shutting down the queues, so workers exit after their current item instead of draining the queued backlog, leaving remaining offsets outstanding and defeating shutdown(immediate=False).",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py",
+        "source_comment_line": 233
+      },
+      {
+        "text": "The exception log for queue worker processing failures includes only the worker ID and omits the available partition and offset, making it impossible to identify which Kafka message failed from the log context.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py",
+        "source_comment_line": 145
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2": {
+    "deputydev-v1": [
+      {
+        "text": "`OptimizedCursorPaginator.get_result` duplicates the base paginator flow for cursor initialization, hit counting, slicing, previous-page handling, cursor construction, and post-query filtering, so fixes or changes to the base implementation can silently diverge; factor the shared flow into the inheritance path and override only the differing offset behavior.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "src/sentry/api/paginator.py",
+        "source_comment_line": 821
+      },
+      {
+        "text": "The comment claims negative offsets are safe because the queryset handles boundary conditions, but the code clamps non-previous offsets to avoid invalid negative queryset slicing; update the documentation to describe the actual clamping behavior instead of misrepresenting queryset negative-index support.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "src/sentry/api/paginator.py",
+        "source_comment_line": 182
+      },
+      {
+        "text": "With advanced pagination enabled, a negative previous-page cursor offset is assigned directly to `start_offset` and used in `queryset[start_offset:stop]`; Django querysets do not support negative slice indices, so such requests will raise at runtime instead of paginating unless the offset is clamped or the branch is removed.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "src/sentry/api/paginator.py",
+        "source_comment_line": 880
+      },
+      {
+        "text": "The Lua buffering script now applies sorted-set commands such as `zcard` and `zunionstore` to keys that previously held Redis Sets, so existing keys or keys written by old instances during a rolling deployment can trigger `WRONGTYPE` and abort the script; add a migration or use a new key namespace to prevent Set and Sorted Set data from overlapping.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "src/sentry/scripts/spans/add-buffer.lua",
+        "source_comment_line": 47
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3": {
+    "deputydev-v1": [
+      {
+        "text": "The comment claims aliasing `should_upsample` as `upsampling_enabled` provides query optimization and caching, but the assignment only renames the eligibility result and adds no optimization or caching behavior; remove the unsupported claim and either use `should_upsample` directly or assign the eligibility result directly to `upsampling_enabled`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "src/sentry/api/endpoints/organization_events_stats.py",
+        "source_comment_line": 226
+      },
+      {
+        "text": "Enabling the `upsampled_count()` rewrite for the errors dataset can emit `upsampled_count() as count` for an errors-dataset request containing `count()`, but `upsampled_count` is only registered in Discover and not in the errors dataset function converter, so the errors query builder cannot resolve the generated function; register `upsampled_count` for the errors dataset or keep this branch disabled until that registration exists.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "src/sentry/api/helpers/error_upsampling.py",
+        "source_comment_line": 112
+      },
+      {
+        "text": "Using a raw substring check for `event.type:error` falsely matches negated, quoted, or mixed event-type queries such as `!event.type:error`, `NOT event.type:error`, or quoted text containing that string, causing the error-only upsampled aggregation transformation to be applied to queries that exclude errors or include non-error events; replace it with query classification that confirms a positive `event.type:error` filter and rejects negated, quoted, or mixed-type cases.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "src/sentry/api/helpers/error_upsampling.py",
+        "source_comment_line": 137
+      },
+      {
+        "text": "The factory checks `client_sample_rate` by truthiness before assigning `sample_rate`, so a numeric `client_sample_rate` of `0.0` is treated as absent and skipped even though it is present and convertible; check explicitly for `None` before converting to preserve the zero sample-rate edge case.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "src/sentry/testutils/factories.py",
+        "source_comment_line": 353
+      }
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/103633": {
+    "deputydev-v1": [
+      {
+        "text": "A cached denial in `permDenialCache` is returned before consulting the positive permission cache or database, so after a permission is later granted, the stale denial can continue returning `Allowed: false` until `shortCacheTTL` expires; this also affects action-level checks whose denial key has empty name and parent segments.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "pkg/services/authz/rbac/service.go",
+        "source_comment_line": 117
+      }
+    ]
+  },
+  "https://github.com/getsentry/sentry/pull/67876": {
+    "deputydev-v1": [
+      {
+        "text": "The call to `get_user_info(payload[\"access_token\"])` is outside any exception handler even though `get_user_info` raises for non-2xx GitHub responses via `raise_for_status`, so invalid tokens, rate limits, or GitHub failures can escape `OAuthLoginView.dispatch` as an unhandled 500 instead of showing the intended installation error page.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "src/sentry/integrations/github/integration.py",
+        "source_comment_line": 434
+      },
+      {
+        "text": "The OAuth token validation only checks whether the `access_token` key exists, so a parsed response containing an empty `access_token=` passes the condition and sends an empty credential to `get_user_info`; the code should validate that the token value is non-empty.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "src/sentry/integrations/github/integration.py",
+        "source_comment_line": 431
+      },
+      {
+        "text": "The new synchronous `get_user_info` request uses an underlying `session.get` without an explicit timeout, so an unresponsive or degraded GitHub API can hold the installation request worker indefinitely.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "src/sentry/integrations/github/integration.py",
+        "source_comment_line": 434
+      },
+      {
+        "text": "The same `Integration` row is queried twice on the path where an integration exists without an `OrganizationIntegration`; the code should retain and reuse the `Integration` loaded for the organization-membership check and inspect its `status` to avoid the extra database round trip.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "src/sentry/integrations/github/integration.py",
+        "source_comment_line": 494
+      },
+      {
+        "text": "`safe_urlopen` is called before the surrounding `try` block, so connection, TLS, timeout, or other network failures while opening the OAuth token endpoint escape as a 500 instead of being converted into the controlled installation error.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "src/sentry/integrations/github/integration.py",
+        "source_comment_line": 423
+      },
+      {
+        "text": "When `Integration.DoesNotExist` is hit, the code advances a new installation before verifying that `github_authenticated_user` matches the GitHub user who installed the app, allowing a supplied installation ID with no existing record to bypass the only authenticated-user versus sender identity check.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "src/sentry/integrations/github/integration.py",
+        "source_comment_line": 482
+      },
+      {
+        "text": "The code directly accesses nested `integration.metadata[\"sender\"][\"login\"]` even though `sender` is only conditionally populated, so an existing integration missing `sender` or `login` raises `KeyError` instead of returning the controlled validation error.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 6,
+        "source_comment_path": "src/sentry/integrations/github/integration.py",
+        "source_comment_line": 503
+      }
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/32918": {
+    "deputydev-v1": [
+      {
+        "text": "Cleanup is registered with the literal alias \"alias\" instead of the actual IDP aliases created by the test (\"idp-alias-\" + i and \"idp-alias-20\"), so the cleanup callbacks target non-existent providers and leave the created identity providers in the test realm, potentially contaminating subsequent tests.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/organization/cache/OrganizationCacheTest.java",
+        "source_comment_line": 381
+      }
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/94942": {
+    "deputydev-v1": [
+      {
+        "text": "`enableSqlExpressions` performs a feature-flag lookup but returns `false` on both paths, making the lookup dead code and misleadingly suggesting the SQL-expression security kill switch can be enabled; this obscures that the permanent disablement is intentional and increases the risk that a future maintainer accidentally re-enables the path.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "pkg/expr/reader.go",
+        "source_comment_line": 194
+      },
+      {
+        "text": "`pkg/expr/sql` exposes a `DB` API whose operations are only failing `not implemented` stubs, but the package lacks documentation explaining that this is an intentional disabled stub and security boundary; without that context, the code looks like unfinished scaffolding and makes accidental later implementation more likely.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "pkg/expr/sql/db.go",
+        "source_comment_line": 1
+      }
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/90939": {
+    "deputydev-v1": [
+      {
+        "text": "In non-Dev environments, callers that observe `entryPointAssetsCache == nil` under the read lock do not re-check the cache after acquiring the write lock, so multiple concurrent callers can serially repeat the expensive filesystem or CDN initialization even after the first caller has populated the cache, overwriting the cached result.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "pkg/api/webassets/webassets.go",
+        "source_comment_line": 48
+      }
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/80329": {
+    "deputydev-v1": [
+      {
+        "text": "In cleanup_test.go, the subtest registers t.Cleanup only after creating test annotations and running initial assertions; because the subtests share fakeSQL, if createTestAnnotations or an initial assertion calls t.Fatal before cleanup is registered, teardown is skipped and leftover annotation rows can pollute subsequent subtests.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "pkg/services/annotations/annotationsimpl/cleanup_test.go",
+        "source_comment_line": 116
+      },
+      {
+        "text": "In xorm_store.go, the SQLite delete path rebuilds the complete values string with fmt.Sprintf on every iteration, repeatedly copying the growing ID list; for large batches such as 32,767 IDs this causes O(n²) copying and allocations, hurting performance.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "pkg/services/annotations/annotationsimpl/xorm_store.go",
+        "source_comment_line": 609
+      },
+      {
+        "text": "In xorm_store.go cleanup diagnostics, successful cleanup batches are logged with Error-level calls and pre-delete logs include the complete ids slice and raw condition; configured batches can contain tens of thousands of IDs, causing oversized false error events, log noise, and exposure of row IDs and query details even when no error occurred.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "pkg/services/annotations/annotationsimpl/xorm_store.go",
+        "source_comment_line": 534
+      },
+      {
+        "text": "In cleanup.go, changing the shared cleanup ticker from every 10 minutes to every minute makes every cleanup job—not only annotation cleanup—eligible to run up to ten times as often, increasing database work across snapshots, dashboard versions, images, annotations, and other cleanup jobs.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "pkg/services/cleanup/cleanup.go",
+        "source_comment_line": 77
+      }
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/90045": {
+    "deputydev-v1": [
+      {
+        "text": "The asynchronous legacy writes in Create, Delete, Update, and DeleteCollection derive their 10-second timeout context from the request context, so request completion or cancellation can cancel the fire-and-forget legacy operation before its timeout and leave legacy storage stale; the timeout should be based on a context detached from request cancellation.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "source_comment_line": 51
+      },
+      {
+        "text": "The asynchronous d.Legacy.DeleteCollection call is measured with startLegacy but recorded via recordStorageDuration, which misroutes legacy latency and failures into the storage histogram and leaves the legacy DeleteCollection metric series incomplete; it should use recordLegacyDuration.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "source_comment_line": 166
+      },
+      {
+        "text": "The Delete method lacks a leading comment describing that it deletes from Storage synchronously and schedules the LegacyStorage deletion asynchronously, making it inconsistent with the documentation pattern used by the other DualWriterMode3 methods.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "source_comment_line": 94
+      },
+      {
+        "text": "The asynchronous legacy writes in Create, Delete, Update, and DeleteCollection capture returned errors but only record them as metric labels, discarding them without logging; legacy divergence can therefore occur without a corresponding production log entry.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "source_comment_line": 55
+      },
+      {
+        "text": "The d.Storage.Update error path uses startStorage but records the failure through recordLegacyDuration, misattributing storage failures and latency to the legacy histogram; it should use recordStorageDuration.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "source_comment_line": 129
+      },
+      {
+        "text": "Create, Delete, Update, and DeleteCollection launch asynchronous legacy-write goroutines without any shared concurrency limit, so bursts of successful write requests can create many concurrent legacy calls and retain request objects and callbacks until the calls finish or time out; concurrency should be bounded with a semaphore or worker queue.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "source_comment_line": 50
+      },
+      {
+        "text": "Delete enriches a log variable with name, kind, and method fields but then stores the base d.Log in the context, so downstream logging that retrieves its logger from ctx loses those fields; the enriched log should be placed into the context.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 6,
+        "source_comment_path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "source_comment_line": 97
+      },
+      {
+        "text": "The d.Storage.Create error path uses startStorage but records the failure through recordLegacyDuration, creating a legacy metric observation even though no legacy operation ran; it should use recordStorageDuration.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 7,
+        "source_comment_path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "source_comment_line": 45
+      },
+      {
+        "text": "The Delete success path passes the per-object name as the third metric argument even though that argument maps to the kind label, creating a separate histogram series for each deleted object and preventing kind-level aggregation; it should pass options.Kind.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 8,
+        "source_comment_path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "source_comment_line": 106
+      },
+      {
+        "text": "The Create, Delete, Update, and DeleteCollection tests do not synchronize with the asynchronous legacy operation or assert mock expectations before returning, so a legacy goroutine that never runs can still leave the tests passing and the mode-3 legacy-write behavior unverified.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 9,
+        "source_comment_path": "pkg/apiserver/rest/dualwriter_mode3_test.go",
+        "source_comment_line": 63
+      }
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/106778": {
+    "deputydev-v1": [
+      {
+        "text": "In AlertRuleMenu.tsx, the Prometheus ability path can set canPause or canDelete to true when only promRule is available, but the menu rendering guards still require rulerRule: Pause checks rulerRuleType.grafana.rule(rulerRule), and Delete checks canDelete && rulerRule. As a result, Pause and Delete are treated as available by permissions but are not exposed for prom-only rules in the new list path.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "public/app/features/alerting/unified/components/rule-viewer/AlertRuleMenu.tsx",
+        "source_comment_line": 87
+      },
+      {
+        "text": "In useAbilities.ts, the `// duplicate` comment above `const { isEditable, isRemovable, loading } = useIsGrafanaPromRuleEditable(rule);` is unclear because it does not identify what is duplicated or explain any intended follow-up, obscuring the purpose of the permission check for future maintainers.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "public/app/features/alerting/unified/hooks/useAbilities.ts",
+        "source_comment_line": 289
+      },
+      {
+        "text": "In useAbilities.ts, `useAllGrafanaPromRuleAbilities` subscribes to the same folder twice for each rule row: `useIsGrafanaPromRuleEditable(rule)` calls `useFolder(folderUid)`, and `useCanSilenceInFolder(folderUid)` reaches another `useFolder(folderUid)` through `useFolderPermissions`. This creates redundant per-row folder subscriptions and unnecessary work in the rule list.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "public/app/features/alerting/unified/hooks/useAbilities.ts",
+        "source_comment_line": 293
+      },
+      {
+        "text": "In RuleActionsButtons.V2.tsx, the warning emitted when identifier construction fails is identical for every failure and does not include context from `groupIdentifier`, `rule`, or `promRule`, so production logs cannot identify which rule or group caused the conversion failure.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "public/app/features/alerting/unified/rule-list/components/RuleActionsButtons.V2.tsx",
+        "source_comment_line": 146
+      },
+      {
+        "text": "In the new list path, `RuleActionsButtons` receives only `promRule`, so `rulerRuleType.grafana.alertingRule(rule)` is false. Although the menu can grant the Silence action through the Prometheus ability and set `showSilenceDrawer`, the drawer is guarded by the ruler-only condition and also requires a ruler DTO, so the visible Silence action becomes clickable but opens no UI for prom-only rules.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": null,
+        "source_comment_line": null
+      }
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/107534": {
+    "deputydev-v1": []
+  },
+  "https://github.com/grafana/grafana/pull/79265": {
+    "deputydev-v1": [
+      {
+        "text": "`RowsAffected() == 0` after the UPDATE is treated as proof that a new device is blocked by the device limit, but the UPDATE also filters on `updated_at BETWEEN ...`; an existing device with a stored timestamp outside that window affects zero rows and is incorrectly denied with `ErrDeviceLimitReached`, causing authentication denial. The code should distinguish an unknown device from a timestamp mismatch, or update any row matching the existing `device_id`, and only return the limit error when the device is genuinely absent while the configured limit is full.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "pkg/services/anonymous/anonimpl/anonstore/database.go",
+        "source_comment_line": 96
+      },
+      {
+        "text": "`anonymousDeviceExpiration` is independently defined as 30 days in both the API package and `anonstore`, even though the values govern the same expiration policy for device listing and device-limit enforcement; changing only one constant would silently make the listing window and limit-enforcement window inconsistent. The expiration should be defined once and reused in both packages.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "pkg/services/anonymous/anonimpl/api/api.go",
+        "source_comment_line": 18
+      },
+      {
+        "text": "`tagDeviceUI` writes the device marker to the local cache before calling `CreateOrUpdateDevice`; if `CreateOrUpdateDevice` returns `ErrDeviceLimitReached`, the request is blocked but the cache entry remains, so the next request for the same device succeeds from the cache without checking the store and bypasses device-limit enforcement. The device should only be cached after `CreateOrUpdateDevice` succeeds.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "pkg/services/anonymous/anonimpl/impl.go",
+        "source_comment_line": 147
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9": {
+    "deputydev-v1": [
+      {
+        "text": "`FallbackLocaleList` subclasses `Hash` while overriding `[]` to always compute a fallback locale list instead of reading Hash storage, so values stored through inherited Hash operations cannot be retrieved through lookup; this breaks Hash semantics and makes inherited storage operations inconsistent with `[]`, so it should be a plain class exposing only the required `[]` and `ensure_loaded!` methods.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "config/initializers/i18n.rb",
+        "source_comment_line": 12
+      }
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/76186": {
+    "deputydev-v1": [
+      {
+        "text": "`TestLogger.FromContext` returns a fresh `TestLogger`, so calls such as `m.logger.FromContext(ctx).Info(...)` write to a throwaway logger instead of the injected test logger, preventing assertions on the original logger's `InfoLogs`, `WarnLogs`, and other stores from observing context-derived log calls.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "pkg/plugins/log/fake.go",
+        "source_comment_line": 47
+      },
+      {
+        "text": "The new contextual logger middleware has no unit tests covering the endpoint attributes added by each instrumented method, optional user and datasource fields, or delegation behavior when optional `PluginContext` fields are nil.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "pkg/services/pluginsintegration/clientmiddleware/contextual_logger_middleware.go",
+        "source_comment_line": 1
+      },
+      {
+        "text": "`logRequest` relies on `ctx` having already been enriched by `ContextualLoggerMiddleware`, but this implicit middleware-order dependency is undocumented; if the ordering is wrong, emitted log entries lack plugin, endpoint, datasource, and user attributes that were previously added directly in `logRequest`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "pkg/services/pluginsintegration/clientmiddleware/logger_middleware.go",
+        "source_comment_line": 36
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10": {
+    "deputydev-v1": [
+      {
+        "text": "In `update`, an unknown or already-deleted `params[:id]` makes `EmbeddableHost.where(id: params[:id]).first` return `nil`, and passing that nil host into `save_host` dereferences it, causing a `NoMethodError`/500 instead of a not-found response.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "app/controllers/admin/embeddable_hosts_controller.rb",
+        "source_comment_line": 10
+      },
+      {
+        "text": "In `destroy`, an unknown or already-deleted `params[:id]` makes `EmbeddableHost.where(id: params[:id]).first` return `nil`, and calling `nil.destroy` causes a `NoMethodError`/500 instead of a not-found response.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "app/controllers/admin/embeddable_hosts_controller.rb",
+        "source_comment_line": 10
+      },
+      {
+        "text": "`record_for_host` lowercases the stored `host` value but not the supplied `uri.host` parameter, so under a case-sensitive comparison a mixed-case URL host fails to match an otherwise allowed host and valid hosts are rejected.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "app/models/embeddable_host.rb",
+        "source_comment_line": 17
+      },
+      {
+        "text": "The `before_validation` host-normalization callback calls `sub!` on `self.host` without checking that it is present, so saving an `EmbeddableHost` without a host raises `NoMethodError` before format validation can return a normal validation error.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "app/models/embeddable_host.rb",
+        "source_comment_line": 6
+      },
+      {
+        "text": "The migration interpolates host values from the free-text site setting directly into a raw SQL `INSERT`; a stored host containing a single quote can break the query and alter the SQL being executed, so each host value must be database-quoted or bound.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "db/migrate/20150818190757_create_embeddable_hosts.rb",
+        "source_comment_line": 25
+      },
+      {
+        "text": "The migration accesses `result[0]['id']` without checking whether the category lookup returned a row; if the `embed_category` setting is absent or matches no category, `result[0]` is nil and the migration raises `NoMethodError` before reaching the uncategorized fallback.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "db/migrate/20150818190757_create_embeddable_hosts.rb",
+        "source_comment_line": 9
+      },
+      {
+        "text": "`EmbeddableHost.record_for_host` repeatedly queries with `WHERE lower(host) = ?`, but the table has no matching functional index on `lower(host)`, so each host authorization lookup scans the table.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "db/migrate/20150818190757_create_embeddable_hosts.rb",
+        "source_comment_line": 3
+      },
+      {
+        "text": "The controller spec only asserts inheritance and does not exercise the newly added CRUD behavior or error paths, leaving create, update, destroy, validation failures, authorization, and missing-record responses untested.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 6,
+        "source_comment_path": "spec/controllers/admin/embeddable_hosts_controller_spec.rb",
+        "source_comment_line": 5
+      },
+      {
+        "text": "The fabricator definitions are in files named for the opposite model: `category_fabricator.rb` now defines only an embeddable host after removing the category definitions, making fabricators difficult to locate and breaking the existing file organization.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 7,
+        "source_comment_path": "spec/fabricators/category_fabricator.rb",
+        "source_comment_line": 1
+      },
+      {
+        "text": "The deleted `embeddable_hosts` and `embed_category` setting keys still have translations in non-English server locale files such as `server.es.yml` and `server.fi.yml`, leaving orphaned locale entries that should be removed.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 8,
+        "source_comment_path": null,
+        "source_comment_line": null
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7": {
+    "deputydev-v1": [
+      {
+        "text": "The documentation comment in `app/assets/stylesheets/common/base/discourse.scss` still describes the old table-cell coldmap default as `scale-color($primary, $lightness: 50%)` instead of the new theme-aware `dark-light-choose(...)` value, so it documents the wrong default expression.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "app/assets/stylesheets/common/base/discourse.scss",
+        "source_comment_line": 73
+      },
+      {
+        "text": "The `.topic-meta-data h5 a` color migration changes the light-theme branch from the original `scale-color($primary, $lightness: 30%)` to `70%`; because `dark-light-choose` uses the first branch on light themes, this makes the link substantially lighter than before and reduces its previous contrast instead of limiting the change to dark-theme rendering.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "app/assets/stylesheets/desktop/topic-post.scss",
+        "source_comment_line": 291
+      },
+      {
+        "text": "The `.group-member-info .name` color replacement in desktop and mobile user styles changes the light-theme value from the original `30%` to `50%`, making member names the same lightness as the adjacent `.title` and removing the previous visual hierarchy between name and title.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "app/assets/stylesheets/desktop/user.scss",
+        "source_comment_line": 522
+      },
+      {
+        "text": "The mobile `.custom-message-length` color conversion replaces the original light-theme first branch `70%` with `30%`; because that branch controls light themes, the text becomes much darker and more prominent than before and diverges from the equivalent desktop rule that preserves `70%`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "app/assets/stylesheets/mobile/modal.scss",
+        "source_comment_line": 102
+      },
+      {
+        "text": "The mobile topic-post `h3` color migration changes the light-theme first branch from the original `20%` lightness to `50%`, making the heading substantially lighter on light themes, removing its previous emphasis, and diverging from the equivalent desktop rule that keeps `20%`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "app/assets/stylesheets/mobile/topic-post.scss",
+        "source_comment_line": 182
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8": {
+    "deputydev-v1": [
+      {
+        "text": "`Math.floor(user_count / limit) + 1` overcounts total pages when the member count is an exact multiple of the page size, creating a phantom empty page such as 50 members with a limit of 50 producing two pages; use `Math.ceil(user_count / limit)` instead.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "app/assets/javascripts/admin/controllers/admin-group.js.es6",
+        "source_comment_line": 13
+      },
+      {
+        "text": "`removeMember` is called without handling its returned promise, so server-side deletion failures are discarded and the confirmation closes without notifying the administrator that the member was not removed.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "app/assets/javascripts/admin/controllers/admin-group.js.es6",
+        "source_comment_line": 56
+      },
+      {
+        "text": "`addMembers` is called without a rejection handler, so network or server-side failures receive no user-facing handling and leave the entered usernames in place without explaining that no members were added.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "app/assets/javascripts/admin/controllers/admin-group.js.es6",
+        "source_comment_line": 64
+      },
+      {
+        "text": "The loop calls `User.find_by_username` once for every submitted username, causing N separate user SELECT queries instead of resolving the submitted usernames with one bulk query before adding the users to the group.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "app/controllers/admin/groups_controller.rb",
+        "source_comment_line": 71
+      },
+      {
+        "text": "The caller-provided `limit` is passed directly to the query without a maximum, so a request such as `limit=100000` can fetch and serialize a very large group in one response, imposing excessive database, memory, and serialization work.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "app/controllers/groups_controller.rb",
+        "source_comment_line": 22
+      },
+      {
+        "text": "The controller spec uses PUT for `remove_member` even though the production route exposes the action through DELETE, so the test can pass by invoking the named action directly without exercising the route's DELETE method contract.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "spec/controllers/admin/groups_controller_spec.rb",
+        "source_comment_line": 115
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3": {
+    "deputydev-v1": [
+      {
+        "text": "`rejectedEmails` is initialized as a mutable array on the controller prototype, so controller instances share rejected-address state; an email pushed by one instance can leak to another and cause that other instance to reject it.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "app/assets/javascripts/discourse/controllers/create_account_controller.js",
+        "source_comment_line": 17
+      },
+      {
+        "text": "The blocklist lookup compares the raw email string directly, so case-sensitive comparison can let differently cased versions of a blocked address, such as `BAD@spamclub.com` versus stored `bad@spamclub.com`, bypass the block.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "app/models/blocked_email.rb",
+        "source_comment_line": 12
+      },
+      {
+        "text": "The `match_count` update uses a non-atomic read-modify-write, so concurrent requests can both read the same count, save the same incremented value, and lose one recorded match.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "app/models/blocked_email.rb",
+        "source_comment_line": 14
+      },
+      {
+        "text": "`BlockedEmail.should_block?` updates `match_count` and `last_match_at` when called from `validate_each`, so repeated validations during one account creation attempt can mutate statistics multiple times and overcount actual registration attempts.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "lib/validators/email_validator.rb",
+        "source_comment_line": 13
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5": {
+    "deputydev-v1": [
+      {
+        "text": "Converting `.small-action` to a flex container makes the existing `float: left` on the direct `.topic-avatar` child and `float: right` on the direct `button` child ineffective, so these declarations no longer control avatar or button placement; remove the dead floats and use flex layout such as `margin-left: auto` if the button must remain flush right.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "app/assets/stylesheets/common/base/topic-post.scss",
+        "source_comment_line": 264
+      },
+      {
+        "text": "The `//Flexbox` section header in `app/assets/stylesheets/common/foundation/mixins.scss` is missing the space used by the file's other section comments, such as `// Text`, and should be changed to `// Flexbox` for consistency.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "app/assets/stylesheets/common/foundation/mixins.scss",
+        "source_comment_line": 98
+      },
+      {
+        "text": "`-ms-align-items` in the `align-items` mixin is not a valid CSS property and is ignored; since the valid prefixed declaration `-ms-flex-align` is already present, every use of the mixin emits a dead invalid vendor-prefixed property that should be removed.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "app/assets/stylesheets/common/foundation/mixins.scss",
+        "source_comment_line": 121
+      },
+      {
+        "text": "The `align-items` mixin is formatted inconsistently with adjacent mixins by using four-space indentation and omitting the space after the colon in `align-items:$alignment`; format the block consistently with the rest of the file.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "app/assets/stylesheets/common/foundation/mixins.scss",
+        "source_comment_line": 117
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6": {
+    "deputydev-v1": [
+      {
+        "text": "Each user serialization redundantly parses the same website URL once before branching and again in branches that append the path, causing unnecessary URI parsing and allocations from unchanged input.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "app/serializers/user_serializer.rb",
+        "source_comment_line": 138
+      },
+      {
+        "text": "The sibling-subdomain logic repeatedly calls split on the same website and Discourse host values while comparing component lengths and suffixes, causing unnecessary allocation of equivalent host-component arrays on each serialization.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "app/serializers/user_serializer.rb",
+        "source_comment_line": 138
+      },
+      {
+        "text": "The equal-depth sibling-subdomain branch is untested: current serializer specs cover unrelated domains, exact matches, and parent domains, but do not exercise the elsif path where suffix comparison matches or does not match, leaving both return paths unprotected.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "spec/serializers/user_serializer_spec.rb",
+        "source_comment_line": 76
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4": {
+    "deputydev-v1": [
+      {
+        "text": "The message sender is validated with `discourseUrl.indexOf(e.origin)` instead of strict equality against the exact origin derived from `discourseUrl`, so a non-equal origin whose serialized value appears as a substring of `discourseUrl` can pass the check and set the iframe height.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "app/assets/javascripts/embed.js",
+        "source_comment_line": 17
+      },
+      {
+        "text": "`i.content` is used without being guarded before calling `scrub`; feed items that omit `content` raise `NoMethodError`, abort processing of the remaining items, and the polling job has retries disabled.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "app/jobs/scheduled/poll_feed.rb",
+        "source_comment_line": 35
+      },
+      {
+        "text": "Feed item HTML is passed to `TopicEmbed.import` after only `CGI.unescapeHTML`, which decodes entities but does not sanitize; because the imported value is later stored as `raw_html` with validations skipped and rendered with `raw`, attacker-controlled scripts or event-handler attributes from the feed can persist and render.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "app/jobs/scheduled/poll_feed.rb",
+        "source_comment_line": 35
+      },
+      {
+        "text": "`url` is fetched with unrestricted `open` after only a host check, without validating resolved addresses or redirect destinations against private, loopback, and link-local ranges or restricting scheme and redirects, so the server-side fetch can reach internal endpoints.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "app/models/topic_embed.rb",
+        "source_comment_line": 48
+      },
+      {
+        "text": "`url` is interpolated unescaped into the imported-from link's `href` attribute and link text; the HTTP-prefix check does not exclude quote characters, so a crafted URL can break out of the single-quoted `href` and inject attributes or markup that is stored as raw HTML and later rendered with `raw`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "app/models/topic_embed.rb",
+        "source_comment_line": 13
+      },
+      {
+        "text": "`open(url).read` performs an unbounded remote read without explicit open/read timeouts or a maximum response size, buffering the full response before Readability processes it, so a slow or very large response can occupy the worker and consume excessive memory.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "app/models/topic_embed.rb",
+        "source_comment_line": 48
+      },
+      {
+        "text": "The template uses `<%- end if %>` with a trailing `if` that has no modifier condition, generating invalid Ruby and causing the embed template to raise a syntax error instead of rendering the embedded topic view.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 6,
+        "source_comment_path": "app/views/embed/best.html.erb",
+        "source_comment_line": 6
+      },
+      {
+        "text": "The full `request.referer` is interpolated directly into a JavaScript string for the `postMessage` target origin; a full path is not a valid target origin, and an unescaped quote in the referer can break out of the string, which can prevent resize-message delivery or inject JavaScript into the iframe response.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 7,
+        "source_comment_path": "app/views/layouts/embed.html.erb",
+        "source_comment_line": 11
+      },
+      {
+        "text": "The throttle key is set with `setnx` and then expired in a separate command; if the process stops after `setnx` succeeds but before `expire`, the key remains without an expiration and that URL is throttled permanently.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 8,
+        "source_comment_path": "lib/topic_retriever.rb",
+        "source_comment_line": 27
+      },
+      {
+        "text": "`TopicRetriever` directly executes the scheduled `PollFeed` job on an unknown embed URL when feed polling is enabled, synchronously downloading and parsing the entire feed and importing every item, which blocks the retrieval worker with network and database work.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 9,
+        "source_comment_path": "lib/topic_retriever.rb",
+        "source_comment_line": 41
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1": {
+    "deputydev-v1": [
+      {
+        "text": "Replacing the previous per-type upload size checks with a uniform 10 MB limit means configured limits such as max_<type>_size_kb no longer drive client validation, so attachment and avatar uploads with lower configured limits can be allowed through and the existing test that sets max_image_size_kb to 5 KB no longer rejects a 10 KB file as expected.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "app/assets/javascripts/discourse/lib/utilities.js",
+        "source_comment_line": 182
+      },
+      {
+        "text": "The upload controller ignores the return value of OptimizedImage.downsize and does not verify the final file size before calling Upload.create_for, so failed or insufficient ImageMagick conversions can be treated as successful, repeated unnecessarily, and an image that remains above max_image_size_kb can continue into upload creation.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "app/controllers/uploads_controller.rb",
+        "source_comment_line": 67
+      },
+      {
+        "text": "Defining self.downsize a second time in Ruby replaces the existing five-argument downsize(from, to, max_width, max_height, opts={}) method, so the existing resize_emoji.rb caller that still invokes OptimizedImage.downsize(path, path, 100, 100, opts) will raise ArgumentError at runtime because the surviving method accepts at most four arguments.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "app/models/optimized_image.rb",
+        "source_comment_line": 149
+      }
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2": {
+    "deputydev-v1": [
+      {
+        "text": "The controller/template binding uses the misspelled identifier `stopNotificiationsText`; because the controller defines it and the Handlebars template references it verbatim, this typo becomes the property name consumers must use and will require a coordinated rename later. Rename it to `stopNotificationsText` in both places.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "app/assets/javascripts/discourse/controllers/topic-unsubscribe.js.es6",
+        "source_comment_line": 5
+      },
+      {
+        "text": "`TopicUser.find_by(user_id: current_user.id, topic_id: params[:topic_id])` can return `nil`, but the result is dereferenced with `tu.notification_level` unconditionally, causing a `NoMethodError` and a 500 response when the user/topic pair has no `TopicUser` row. Handle the missing `TopicUser` before reading `tu.notification_level`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "app/controllers/topics_controller.rb",
+        "source_comment_line": 105
+      },
+      {
+        "text": "The `%{respond_instructions}` footer is rendered inside the `context_posts.present?` block and also rendered unconditionally later in the template, so emails with context posts receive duplicate response instructions. Remove the placeholder from the `context_posts` block.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "app/views/email/notification.html.erb",
+        "source_comment_line": 6
+      },
+      {
+        "text": "The unsubscribe endpoint performs a persistent notification-state mutation via GET even though the URL is included in notification emails; automated link followers such as mail scanners or preview tools can therefore change a user's notification level without user intent. Use GET only for a confirmation page and perform the mutation via POST or PATCH.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "config/routes.rb",
+        "source_comment_line": 440
+      }
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/22532": {
+    "deputydev-v1": [
+      {
+        "text": "`updateManyByCredentialId` forwards an empty update object directly to Prisma's `updateMany`, so no field is written and `@updatedAt` is not bumped, leaving `SelectedCalendar.updatedAt` stale after availability is refreshed instead of recording the cache-refresh timestamp.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "packages/app-store/googlecalendar/lib/CalendarService.ts",
+        "source_comment_line": 1024
+      },
+      {
+        "text": "After deleting the cache, `viewer.calendars.connectedCalendars` is not invalidated, so the cached tRPC query result can continue showing the deleted `cacheUpdatedAt` value and stale cache status, unlike the disconnect mutation which explicitly invalidates the query.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "packages/features/apps/components/CredentialActionsDropdown.tsx",
+        "source_comment_line": 42
+      },
+      {
+        "text": "The missing or inaccessible credential path throws a plain `Error`, which tRPC reports as `INTERNAL_SERVER_ERROR`, so an expected ownership lookup failure is exposed to the client with incorrect HTTP/tRPC semantics instead of an explicit `NOT_FOUND` or `UNAUTHORIZED` `TRPCError`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "packages/trpc/server/routers/viewer/calendars/deleteCache.handler.ts",
+        "source_comment_line": 25
+      },
+      {
+        "text": "The script uses BSD/macOS-specific `sed -i ''` syntax when updating an existing `GOOGLE_WEBHOOK_URL`, which GNU sed on Linux interprets incorrectly and can terminate the script instead of updating the environment file.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "scripts/test-gcal-webhooks.sh",
+        "source_comment_line": 68
+      }
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/8330": {
+    "deputydev-v1": [
+      {
+        "text": "In `packages/trpc/server/routers/viewer/slots.ts` line 114, the zero-length date override guard compares two separately constructed Dayjs objects with `===`: `dayjs(date.start).add(...) === dayjs(date.end).add(...)`. Because JavaScript object equality checks identity rather than timestamp value, this condition is always false for distinct Dayjs instances, making the zero-duration override branch unreachable and causing zero-length overrides to be silently ignored; use Dayjs value comparison such as `.isSame()` instead.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "packages/trpc/server/routers/viewer/slots.ts",
+        "source_comment_line": 114
+      },
+      {
+        "text": "In `packages/trpc/server/routers/viewer/slots.ts` line 142, the working-hours fallback computes `end` from `slotStartTime` instead of `slotEndTime`, making `start` and `end` identical. As a result, `end > workingHour.endTime` checks the slot start rather than the actual slot end, so slots that start within working hours but finish after the working-hours boundary are incorrectly allowed.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "packages/trpc/server/routers/viewer/slots.ts",
+        "source_comment_line": 142
+      },
+      {
+        "text": "In `packages/trpc/server/routers/viewer/slots.ts` line 114, the zero-length date override check uses `===` between two separate Dayjs instances. Since distinct objects are never referentially equal even when they represent the same instant, the zero-length override exclusion branch is permanently unreachable, causing zero-length overrides to be ignored; compare the Dayjs values with `.isSame()` instead.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "packages/trpc/server/routers/viewer/slots.ts",
+        "source_comment_line": 114
+      },
+      {
+        "text": "In `packages/trpc/server/routers/viewer/slots.ts` line 142, `const end` in the `checkIfIsAvailable` working-hours fallback is calculated with `slotStartTime` rather than `slotEndTime`. This makes both `start` and `end` represent the slot start, so the `end > workingHour.endTime` guard fails to reject slots whose actual end time extends past the working-hours window.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "packages/trpc/server/routers/viewer/slots.ts",
+        "source_comment_line": 142
+      },
+      {
+        "text": "In `packages/trpc/server/routers/viewer/slots.ts`, the flattened `dateOverrides` array containing overrides for all hosts is passed into `checkIfIsAvailable`, but the date-override predicate does not filter overrides by the current user being checked. In multi-host scenarios such as collective or round-robin events, a date override belonging to host A can incorrectly suppress or activate availability for host B, producing incorrect slot availability results.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "packages/trpc/server/routers/viewer/slots.ts",
+        "source_comment_line": 79
+      },
+      {
+        "text": "In `apps/web/test/lib/getSchedule.test.ts` line 798, the assertion comment `// it should return the same as this is the utc time` is misleading because the test is actually verifying that UTC slot times remain invariant across attendee timezones for date override boundaries. The vague comment obscures the timezone-offset invariant being tested and may cause future maintainers to misread the test as a no-op UTC assertion.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "apps/web/test/lib/getSchedule.test.ts",
+        "source_comment_line": 798
+      },
+      {
+        "text": "In `packages/lib/slots.ts` line 218, `getSlots` repeatedly constructs identical Dayjs values inside the `activeOverrides.flatMap` callback: `dayjs(override.start).utc().add(offset, \"minute\")` is computed twice for `startTime`, and `dayjs(override.end).utc().add(offset, \"minute\")` is computed twice for `endTime`. This creates avoidable parsing/wrapping/allocation overhead for every override across scheduling loops; cache the adjusted start and end Dayjs values once per override.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 6,
+        "source_comment_path": "packages/lib/slots.ts",
+        "source_comment_line": 218
+      },
+      {
+        "text": "In `packages/trpc/server/routers/viewer/slots.ts` line 106, the `dateOverrides.find()` callback in `checkIfIsAvailable` repeatedly constructs the same adjusted Dayjs values for each override during every slot availability check. `dayjs(date.start).add(utcOffset, \"minutes\")` is computed up to three times and `dayjs(date.end).add(utcOffset, \"minutes\")` up to two times per iteration, with a timezone lookup also repeated, creating avoidable O(slots × hosts × overrides) allocation and parsing cost; cache the adjusted start and end values inside the callback.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 7,
+        "source_comment_path": "packages/trpc/server/routers/viewer/slots.ts",
+        "source_comment_line": 106
+      },
+      {
+        "text": "In `packages/trpc/server/routers/viewer/slots.ts` line 142, the working-hours fallback assigns both `start` and `end` using the same `slotStartTime.hour() * 60 + slotStartTime.minute()` expression. This duplicates computation and, more importantly, never uses `slotEndTime`, so the end-time boundary is not actually checked and slots that extend beyond working hours can be accepted.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 8,
+        "source_comment_path": "packages/trpc/server/routers/viewer/slots.ts",
+        "source_comment_line": 142
+      },
+      {
+        "text": "In `packages/trpc/server/routers/viewer/slots.ts` line 114, the equality check `dayjs(date.start).add(...) === dayjs(date.end).add(...)` compares Dayjs object references instead of their time values. Because the two Dayjs instances are separately created, the branch intended to detect zero-length overrides is unreachable, leading to unexpected availability behavior; use `.isSame()` for value comparison.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 9,
+        "source_comment_path": "packages/trpc/server/routers/viewer/slots.ts",
+        "source_comment_line": 114
+      },
+      {
+        "text": "In `packages/trpc/server/routers/viewer/slots.ts` line 142, `const end` in the working-hours fallback is incorrectly calculated from `slotStartTime` rather than `slotEndTime`. This makes the end boundary check compare the slot start against `workingHour.endTime`, so a slot whose start is within working hours but whose actual end exceeds the working-hours boundary will not be excluded.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 10,
+        "source_comment_path": "packages/trpc/server/routers/viewer/slots.ts",
+        "source_comment_line": 142
+      },
+      {
+        "text": "In `packages/trpc/server/routers/viewer/slots.ts` line 87, the `checkIfIsAvailable` `dateOverrides` parameter type only declares `{ start: Date; end: Date }[]` even though the flattened array passed at runtime includes per-user `userId` and `timeZone` fields. Because the function iterates the full flattened array without filtering by the current host/user, an override belonging to one user can incorrectly affect another user's availability in fixed-host or other multi-user scheduling paths.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 11,
+        "source_comment_path": "packages/trpc/server/routers/viewer/slots.ts",
+        "source_comment_line": 87
+      },
+      {
+        "text": "In `packages/trpc/server/routers/viewer/slots.ts` line 87, the `checkIfIsAvailable` `dateOverrides` type omits the `timeZone` and `userId` fields that are added to the flattened date override entries in `getSchedule`. This makes the type out of sync with the actual data shape and prevents the predicate from using per-override timezone or user information for correct conversion and filtering.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 12,
+        "source_comment_path": "packages/trpc/server/routers/viewer/slots.ts",
+        "source_comment_line": 87
+      }
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/14943": {
+    "deputydev-v1": [
+      {
+        "text": "In `scheduleSMSReminders.ts`, the retry-count branch of `prisma.workflowReminder.deleteMany` lacks `method: WorkflowMethods.SMS`, so it deletes any `WorkflowReminder` with `retryCount > 1` regardless of delivery method; this can incorrectly delete reminders belonging to non-SMS workflows even though the surrounding cleanup and scheduling logic is SMS-specific.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "packages/features/ee/workflows/api/scheduleSMSReminders.ts",
+        "source_comment_line": 39
+      },
+      {
+        "text": "The retry count update uses `reminder.retryCount + 1` based on a value fetched before processing, including in the catch path, so overlapping cron executions can read the same stale retry count and overwrite each other with the same incremented value; this lost increment can delay retry-based cleanup because multiple failures are not all recorded.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "packages/features/ee/workflows/api/scheduleSMSReminders.ts",
+        "source_comment_line": 184
+      }
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/22345": {
+    "deputydev-v1": [
+      {
+        "text": "`getAuthorizationConditions()` returns a non-nullable `Prisma.Sql`, so `authConditions` is always truthy; the current `getBaseConditions()` control flow includes filter-only and final `NOTHING_CONDITION` branches that are unreachable under the method contract, misleadingly advertising states that cannot occur and should be simplified to only branch on `filterConditions`.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "packages/lib/server/service/insightsBooking.ts",
+        "source_comment_line": 72
+      },
+      {
+        "text": "Caching regression coverage was removed even though `InsightsBookingService` still memoizes authorization and filter conditions via `cachedAuthConditions` and `cachedFilterConditions`; without equivalent tests using strict reference equality on consecutive calls to the new `Prisma.Sql` results, regressions that rebuild conditions instead of returning the cached object would no longer be detected.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": null,
+        "source_comment_line": null
+      }
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/11059": {
+    "deputydev-v1": [
+      {
+        "text": "The user and app existence checks in the app credential webhook are awaited sequentially even though neither query depends on the other, adding unnecessary serial database latency to every credential sync invocation.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "apps/web/pages/api/webhook/app-credential.ts",
+        "source_comment_line": 34
+      },
+      {
+        "text": "The webhook handler calls `schema.parse(req.body)`, so malformed or missing request bodies throw an unhandled Zod exception and escape as a 500 instead of returning a controlled 400 validation response.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "apps/web/pages/api/webhook/app-credential.ts",
+        "source_comment_line": 31
+      },
+      {
+        "text": "The decrypted credential payload is passed to `JSON.parse` without error handling, so malformed decrypted JSON can cause an uncontrolled exception.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "apps/web/pages/api/webhook/app-credential.ts",
+        "source_comment_line": 57
+      },
+      {
+        "text": "The decrypted credential payload is persisted to `credential.key` after any successful JSON parse without validating that it matches the selected app's expected credential structure, allowing structurally invalid credentials to be stored.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "apps/web/pages/api/webhook/app-credential.ts",
+        "source_comment_line": 57
+      },
+      {
+        "text": "The Zod schema uses computed keys with `toString()`, which creates fixed property names rather than wildcard fields, so it does not validate arbitrary numeric expiry fields or preserve arbitrary token fields as intended in sync mode.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts",
+        "source_comment_line": 5
+      },
+      {
+        "text": "When the OAuth refresh response omits `refresh_token`, the parser substitutes the literal string `\"refresh_token\"`, allowing callers to persist that placeholder over a valid stored refresh token and causing later provider refreshes to fail.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts",
+        "source_comment_line": 26
+      },
+      {
+        "text": "The credential-sync POST identifies a Cal.com user and app but sends no authorization or shared-secret information, leaving the external endpoint without an application-level way to verify that the request originated from this service.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "packages/app-store/_utils/oauth/refreshOAuthTokens.ts",
+        "source_comment_line": 8
+      },
+      {
+        "text": "The OAuth refresh path awaits the external credential-sync `fetch` without an abort timeout, so an unresponsive sync endpoint can indefinitely stall calendar or video operations waiting for token refresh.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 6,
+        "source_comment_path": "packages/app-store/_utils/oauth/refreshOAuthTokens.ts",
+        "source_comment_line": 8
+      },
+      {
+        "text": "In sync mode, `refreshOAuthTokens` returns a raw Fetch `Response` even though callers expect provider-specific token objects, causing sync-mode refreshes to produce undefined fields or runtime errors when callers read shapes such as `.data` or `HubspotToken` fields.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 7,
+        "source_comment_path": "packages/app-store/_utils/oauth/refreshOAuthTokens.ts",
+        "source_comment_line": 14
+      },
+      {
+        "text": "The credential-sync helper returns the Fetch response without checking `response.ok`, so HTTP 4xx or 5xx responses are treated as successful token refreshes and only fail later while parsing or using the response.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 8,
+        "source_comment_path": "packages/app-store/_utils/oauth/refreshOAuthTokens.ts",
+        "source_comment_line": 14
+      },
+      {
+        "text": "Google Calendar assigns the entire `{ success: true, data: ... }` result from `parseRefreshTokenResponse` directly to the credential `key`, causing Prisma to persist the validation wrapper instead of the Google credential object.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 9,
+        "source_comment_path": "packages/app-store/googlecalendar/lib/CalendarService.ts",
+        "source_comment_line": 97
+      },
+      {
+        "text": "The Salesforce refresh code contains a failure branch for a `parseRefreshTokenResponse` result even though the helper throws on validation failure and only returns successful results, making that branch unreachable and inconsistent with the helper contract.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 10,
+        "source_comment_path": "packages/app-store/salesforce/lib/CalendarService.ts",
+        "source_comment_line": 92
+      },
+      {
+        "text": "Zoho Bigin passes `credentialId` as the helper's third argument, which is serialized as `calcomUserId`, so sync-mode requests send a credential row ID in the field reserved for the Cal.com user ID and identify the wrong user.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 11,
+        "source_comment_path": "packages/app-store/zoho-bigin/lib/CalendarService.ts",
+        "source_comment_line": 93
+      }
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/7232": {
+    "deputydev-v1": [
+      {
+        "text": "In `handleCancelBooking.ts`, reminder cancellations are started inside nested `forEach` loops without awaiting the asynchronous `deleteScheduledEmailReminder` and `deleteScheduledSMSReminder` calls, so the booking cancellation handler can continue and respond before reminder rows have been cancelled or deleted.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "packages/features/bookings/lib/handleCancelBooking.ts",
+        "source_comment_line": 485
+      },
+      {
+        "text": "In `handleNewBooking.ts`, the reschedule flow uses `forEach` without awaiting the asynchronous reminder-cancellation helpers, so the surrounding `try/catch` can finish immediately and the reschedule can proceed while the previous booking's email or SMS reminders remain active.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "packages/features/bookings/lib/handleNewBooking.ts",
+        "source_comment_line": 966
+      },
+      {
+        "text": "In `scheduleEmailReminders.ts`, wrapping the entire reminder-cancellation loop in one `try/catch` means a single `client.request` failure aborts the rest of the batch, leaves later reminders active, and prevents already accumulated database deletions from being awaited.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "packages/features/ee/workflows/api/scheduleEmailReminders.ts",
+        "source_comment_line": 53
+      },
+      {
+        "text": "In `WorkflowStepContainer.tsx`, the phone verification placeholder `\"Verification code\"` and button label `\"Verify\"` are hard-coded instead of using the `t(...)` i18n function, so those user-facing strings will always appear in English regardless of the user's locale.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "packages/features/ee/workflows/components/WorkflowStepContainer.tsx",
+        "source_comment_line": 439
+      },
+      {
+        "text": "In `emailReminderManager.ts`, the `immediateDelete` branch cancels the SendGrid batch and then returns without deleting or updating the local `workflowReminder` row, leaving stale reminder records after an immediate deletion request.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "packages/features/ee/workflows/lib/reminders/emailReminderManager.ts",
+        "source_comment_line": 222
+      }
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/14740": {
+    "deputydev-v1": [
+      {
+        "text": "`ZAddGuestsInputSchema` is defined inside `AddGuestsDialog` even though it has no dependency on component state or props, causing the same static Zod schema object to be reconstructed on every render instead of being hoisted to module scope.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "apps/web/components/dialog/AddGuestsDialog.tsx",
+        "source_comment_line": 26
+      },
+      {
+        "text": "The authorization check combines `isTeamAdmin` and `isTeamOwner` with `&&`, but `isTeamAdmin` already accepts ADMIN and OWNER while `isTeamOwner` accepts only OWNER, so ADMIN-only members who are neither organizer nor attendee are incorrectly denied permission to add guests.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+        "source_comment_line": 47
+      },
+      {
+        "text": "`guest` is not normalized before checking against lowercase blacklisted guest emails or existing attendees, so case variants can bypass the blacklist and case variants of existing attendee emails can be inserted as duplicate attendees.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+        "source_comment_line": 77
+      },
+      {
+        "text": "The organizer translation is awaited only after all attendee translations complete even though the operations are independent, adding avoidable serial translation latency and delaying calendar processing.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+        "source_comment_line": 120
+      },
+      {
+        "text": "`sendAddGuestsEmails` is called with the original `guests` input instead of `uniqueGuests`, so email-template routing can classify an already-existing submitted attendee as newly added and send the full booking-confirmation template instead of the guests-added notification.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+        "source_comment_line": 168
+      },
+      {
+        "text": "The email-delivery catch block logs only a context-free message and discards the caught error, leaving no stack, error details, booking ID, or user context to diagnose a swallowed `sendAddGuestsEmails` failure.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+        "source_comment_line": 170
+      }
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/10600": {
+    "deputydev-v1": [
+      {
+        "text": "In `apps/web/components/settings/EnableTwoFactorModal.tsx`, the Copy button calls `navigator.clipboard.writeText()` without awaiting or catching the returned Promise, so the success toast is shown even when clipboard copying fails due to an unavailable Clipboard API, non-HTTPS context, denied permission, or another rejection, misleading users into believing their backup codes were copied.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "apps/web/components/settings/EnableTwoFactorModal.tsx",
+        "source_comment_line": 278
+      },
+      {
+        "text": "In `packages/features/auth/lib/next-auth-options.ts`, password verification is skipped whenever `credentials.totpCode` is non-empty because the password check is gated by `if (user.password && !credentials.totpCode)`. If an attacker submits both `totpCode` and `backupCode`, the presence of `totpCode` skips the password check while the backup-code branch can still grant access, allowing a crafted login request to bypass password verification.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "packages/features/auth/lib/next-auth-options.ts",
+        "source_comment_line": 131
+      },
+      {
+        "text": "In `apps/web/components/auth/BackupCode.tsx`, the default-exported component is declared as `TwoFactor` even though the file and imports identify it as `BackupCode`, making the component purpose unclear and causing confusing IDE name resolution or stack traces.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "apps/web/components/auth/BackupCode.tsx",
+        "source_comment_line": 7
+      },
+      {
+        "text": "In `apps/web/pages/api/auth/two-factor/totp/disable.ts`, the `console.error` message says it \"cannot proceed with backup code login\" even though the endpoint is disabling 2FA, which can mislead engineers triaging logs for this endpoint.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "apps/web/pages/api/auth/two-factor/totp/disable.ts",
+        "source_comment_line": 50
+      },
+      {
+        "text": "In `apps/web/components/settings/EnableTwoFactorModal.tsx`, `URL.createObjectURL` is used to create a backup-code download URL after setup, but the URL is only revoked on the next setup attempt and not when the modal is closed, canceled, reset, or unmounted, so the Blob URL and backup-code text remain in memory for the lifetime of the page and can accumulate across modal openings.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "apps/web/components/settings/EnableTwoFactorModal.tsx",
+        "source_comment_line": 99
+      },
+      {
+        "text": "The duplicated `apps/web/components/security/TwoFactorAuthAPI.ts` still exposes the old two-argument `disable(password, code)` signature while `apps/web/components/settings/TwoFactorAuthAPI.ts` was updated to accept a backup code, so the security-page disable flow cannot send `backupCode` to the API and users there cannot disable 2FA with a backup code.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "apps/web/components/settings/TwoFactorAuthAPI.ts",
+        "source_comment_line": 22
+      },
+      {
+        "text": "The duplicated `apps/web/components/security/DisableTwoFactorModal.tsx` remains separate from the updated settings modal and still calls `TwoFactorAuthAPI.disable(password, totpCode)` without backup-code support, creating a maintenance hazard where security-critical 2FA changes must be applied to multiple divergent component copies.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "apps/web/components/settings/TwoFactorAuthAPI.ts",
+        "source_comment_line": 22
+      },
+      {
+        "text": "In `apps/web/components/settings/EnableTwoFactorModal.tsx`, `backupCodes` is initialized with `useState([])` instead of an explicit `useState<string[]>([])`, which removes or weakens TypeScript safety for downstream uses such as `backupCodes.map(formatBackupCode)` and clipboard serialization, allowing API response shape or value mismatches to go uncaught by the compiler.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 6,
+        "source_comment_path": "apps/web/components/settings/EnableTwoFactorModal.tsx",
+        "source_comment_line": 64
+      },
+      {
+        "text": "In `apps/web/components/settings/EnableTwoFactorModal.tsx`, `resetState()` resets the password, error message, and step but does not clear `backupCodes`, so after closing and reopening the modal while it remains mounted, stale backup codes from a previous setup session can remain in state and be displayed.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 7,
+        "source_comment_path": "apps/web/components/settings/EnableTwoFactorModal.tsx",
+        "source_comment_line": 71
+      },
+      {
+        "text": "In `apps/web/components/settings/EnableTwoFactorModal.tsx`, `resetState()` does not revoke or clear `backupCodesUrl`, so the object URL created for backup-code download is not released when the modal is closed without a new setup attempt, leaking the Blob URL and its backup-code data.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 7,
+        "source_comment_path": "apps/web/components/settings/EnableTwoFactorModal.tsx",
+        "source_comment_line": 71
+      },
+      {
+        "text": "In `apps/web/components/settings/EnableTwoFactorModal.tsx`, the Copy button ignores the Promise returned by `navigator.clipboard.writeText()` and always shows the success toast, so if clipboard access is denied, unavailable, blocked in an iframe, or otherwise rejects, users are falsely told their backup codes were copied.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 8,
+        "source_comment_path": "apps/web/components/settings/EnableTwoFactorModal.tsx",
+        "source_comment_line": 276
+      },
+      {
+        "text": "The duplicated `apps/web/components/security/DisableTwoFactorModal.tsx` and `apps/web/components/security/TwoFactorAuthAPI.ts` were not migrated to the new backup-code disable flow and still use the old two-argument `TwoFactorAuthAPI.disable(password, totpCode)` call, so users accessing 2FA disable through the security settings page cannot send a backup code and cannot use backup codes to disable 2FA.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 9,
+        "source_comment_path": null,
+        "source_comment_line": null
+      }
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/10967": {
+    "deputydev-v1": [
+      {
+        "text": "When `externalCalendarId` is falsy, the Google Calendar fallback searches using the absent `externalCalendarId` instead of selecting the event's destination calendar, so `updateEvent` can pass an undefined calendar ID to Google and deletes can fall back to `primary` rather than the stored destination calendar.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "packages/app-store/googlecalendar/lib/CalendarService.ts",
+        "source_comment_line": 256
+      },
+      {
+        "text": "If a calendar reference credential is missing from `this.calendarCredentials` and the database lookup does not return a usable credential, `credential` remains undefined but `updateEvent` is still called, causing the update path to fail when it accesses credential data.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "packages/core/EventManager.ts",
+        "source_comment_line": 542
+      },
+      {
+        "text": "When `evt.destinationCalendar` is null, undefined, or empty, `mainHostDestinationCalendar` is undefined and accessing `.integration` crashes Google Meet booking creation before the Daily fallback can run.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "packages/core/EventManager.ts",
+        "source_comment_line": 119
+      },
+      {
+        "text": "Creating an event for multiple destination calendars with credentials missing from `this.calendarCredentials` awaits `findUnique` inside the loop, causing N uncached host credentials to incur N sequential database round trips.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "packages/core/EventManager.ts",
+        "source_comment_line": 344
+      },
+      {
+        "text": "Updating calendar references with credentials missing from `this.calendarCredentials` awaits `findUnique` inside the loop, so a collective reschedule with N uncached credentials incurs N sequential database round trips.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "packages/core/EventManager.ts",
+        "source_comment_line": 517
+      },
+      {
+        "text": "Cancelling a collective booking awaits a credential lookup inside the loop over calendar references, so every credential missing from `bookingToDelete.user.credentials` adds one sequential database round trip and increases cancellation latency.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 5,
+        "source_comment_path": "packages/features/bookings/lib/handleCancelBooking.ts",
+        "source_comment_line": 433
+      },
+      {
+        "text": "When neither the event type nor organizer has a destination calendar, `evt.destinationCalendar` is initialized to null and optional chaining skips appending team destination calendars, causing collective events to silently omit all team-host destination calendars.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 6,
+        "source_comment_path": "packages/features/bookings/lib/handleNewBooking.ts",
+        "source_comment_line": 1078
+      }
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/8087": {
+    "deputydev-v1": [
+      {
+        "text": "In packages/app-store/vital/lib/reschedule.ts at line 125, `forEach` is used with an async callback for calendar/video deletions, so the returned deletion promises are discarded; `Reschedule` can return success before cleanup completes, and rejections from the callbacks bypass the surrounding `try/catch` and may become unhandled.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 0,
+        "source_comment_path": "packages/app-store/vital/lib/reschedule.ts",
+        "source_comment_line": 125
+      },
+      {
+        "text": "In packages/app-store/wipemycalother/lib/reschedule.ts at line 125, `forEach` is used with an async callback for calendar/video deletions, so the deletion promises are not awaited; `Reschedule` can proceed or complete before cleanup finishes, and rejected callback promises bypass the surrounding `try/catch` and remain unobserved.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 1,
+        "source_comment_path": "packages/app-store/wipemycalother/lib/reschedule.ts",
+        "source_comment_line": 125
+      },
+      {
+        "text": "In packages/core/videoClient.ts at line 24, awaiting each app module resolution inside a `for...of` loop serializes independent video credential processing; users with multiple video credentials incur the sum of all module-resolution delays instead of resolving them concurrently.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 2,
+        "source_comment_path": "packages/core/videoClient.ts",
+        "source_comment_line": 24
+      },
+      {
+        "text": "In packages/features/bookings/lib/handleCancelBooking.ts at line 460, an async `forEach` for recurring calendar cleanup is not awaited, so callbacks can pause at `getCalendar` while downstream `Promise.all(apiDeletes)` runs before those callbacks have pushed their deletion promises; cancellation can proceed without waiting for recurring calendar deletions, and failures from those deletions are not observed in the handler.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 3,
+        "source_comment_path": "packages/features/bookings/lib/handleCancelBooking.ts",
+        "source_comment_line": 460
+      },
+      {
+        "text": "In packages/trpc/server/routers/viewer/bookings.tsx at line 553, `forEach` discards async callback promises for calendar/video deletion, so the mutation can send the reschedule email and return success before old integration events are removed, and rejected deletion promises are not propagated through the mutation.",
+        "path": null,
+        "line": null,
+        "source": "extracted",
+        "source_comment_index": 4,
+        "source_comment_path": "packages/trpc/server/routers/viewer/bookings.tsx",
+        "source_comment_line": 553
+      }
+    ]
+  }
+}
diff --git a/offline/results/gpt-5.5/dedup_groups.json b/offline/results/gpt-5.5/dedup_groups.json
new file mode 100644
index 0000000..be0bbbf
--- /dev/null
+++ b/offline/results/gpt-5.5/dedup_groups.json
@@ -0,0 +1,812 @@
+{
+  "https://github.com/keycloak/keycloak/pull/37429": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/37634": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ]
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/38446": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ]
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/36882": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ]
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/36880": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ]
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/37038": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/33832": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
+    ]
+  },
+  "https://github.com/keycloak/keycloak/pull/40940": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
+    ]
+  },
+  "https://github.com/getsentry/sentry/pull/93824": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
+    ]
+  },
+  "https://github.com/getsentry/sentry/pull/80168": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ]
+    ]
+  },
+  "https://github.com/getsentry/sentry/pull/80528": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
+    ]
+  },
+  "https://github.com/getsentry/sentry/pull/77754": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
+    ]
+  },
+  "https://github.com/getsentry/sentry/pull/67876": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ],
+      [
+        6
+      ]
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/94942": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ]
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/80329": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/90045": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ],
+      [
+        6
+      ],
+      [
+        7
+      ],
+      [
+        8
+      ],
+      [
+        9
+      ]
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/106778": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/79265": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
+    ]
+  },
+  "https://github.com/grafana/grafana/pull/76186": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ],
+      [
+        6
+      ],
+      [
+        7
+      ],
+      [
+        8
+      ],
+      [
+        9
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ],
+      [
+        6
+      ],
+      [
+        7
+      ],
+      [
+        8
+      ],
+      [
+        9
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
+    ]
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/22532": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/8330": {
+    "deputydev-v1": [
+      [
+        0,
+        2,
+        9
+      ],
+      [
+        1,
+        3,
+        8,
+        10
+      ],
+      [
+        4,
+        11
+      ],
+      [
+        5
+      ],
+      [
+        6
+      ],
+      [
+        7
+      ],
+      [
+        12
+      ]
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/14943": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ]
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/11059": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ],
+      [
+        6
+      ],
+      [
+        7
+      ],
+      [
+        8
+      ],
+      [
+        9
+      ],
+      [
+        10
+      ],
+      [
+        11
+      ],
+      [
+        12
+      ]
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/7232": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/14740": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ]
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/10600": {
+    "deputydev-v1": [
+      [
+        0,
+        10
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4,
+        9
+      ],
+      [
+        5
+      ],
+      [
+        6
+      ],
+      [
+        7
+      ],
+      [
+        8
+      ],
+      [
+        11
+      ]
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/10967": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ],
+      [
+        6
+      ]
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/8087": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
+    ]
+  },
+  "https://github.com/getsentry/sentry/pull/95633": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
+    ]
+  },
+  "https://github.com/calcom/cal.com/pull/22345": {
+    "deputydev-v1": [
+      [
+        0
+      ],
+      [
+        1
+      ]
+    ]
+  }
+}
diff --git a/offline/results/gpt-5.5/evaluations.json b/offline/results/gpt-5.5/evaluations.json
new file mode 100644
index 0000000..e0f5c35
--- /dev/null
+++ b/offline/results/gpt-5.5/evaluations.json
@@ -0,0 +1,2624 @@
+{
+  "https://github.com/keycloak/keycloak/pull/37429": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The translation is in Italian instead of Lithuanian. This should be translated to Lithuanian to match the file's locale (messages_lt.properties).",
+          "severity": "Medium",
+          "matched_candidate": "messages_lt.properties assigns the Italian sentence \"Installa una delle seguenti applicazioni sul tuo cellulare:\" to loginTotpStep1, so Lithuanian users would see Italian text during TOTP setup instead of an appropriate Lithuanian translation.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same issue: an Italian string is present in messages_lt.properties, causing Lithuanian users to see Italian text instead of Lithuanian for loginTotpStep1."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "VerifyMessageProperties initializes immutable HTML policy factories and recompiles the reusable HTML_TAGS regex for every instance even though they do not depend on instance state, so creating a verifier per properties file repeatedly rebuilds the same policies and pattern instead of declaring them private static final constants."
+        },
+        {
+          "candidate": "The anchor-mismatch diagnostic does not include the property key or file, so builds processing many resource bundles cannot identify which translated property produced the missing-anchor message, unlike the nearby illegal-HTML diagnostic."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese file (zh_CN), which is likely incorrect for this locale. Please verify the locale‑appropriate translation.",
+          "severity": "Medium"
+        },
+        {
+          "golden_comment": "The anchor sanitization logic has a potential issue where it consumes English matcher groups without proper validation. If the translated text has more anchor tags than the English text, this could lead to incorrect validation results.",
+          "severity": "Low"
+        },
+        {
+          "golden_comment": "The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 3,
+      "total_scored_candidates": 3,
+      "total_golden": 4,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 2,
+      "fn": 3,
+      "errors_count": 0,
+      "precision": 0.3333333333333333,
+      "recall": 0.25,
+      "tool": "deputydev-v1",
+      "repo_name": "keycloak__keycloak__deputydev-v5__PR37429__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR37429__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/keycloak/keycloak/pull/37634": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Wrong parameter in null check (grantType vs. rawTokenId)",
+          "severity": "Critical",
+          "matched_candidate": "The constructor repeats the null check for `grantType` while the error message refers to `rawTokenId`, so `rawTokenId` is stored without validation and an `AccessTokenContext` can be constructed with that required field unset.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies that the null check mistakenly checks grantType again while intending to validate rawTokenId, matching the golden issue of using the wrong parameter in the null check."
+        },
+        {
+          "golden_comment": "In isAccessTokenId, the substring for the grant shortcut and the equality check look inverted: the grant shortcut occupies indices 4–5 (substring(4,6)), and a match should return true (combined with UUID check), not false.",
+          "severity": "High",
+          "matched_candidate": "The event matcher extracts the grant shortcut from the wrong substring and uses the equality check with the wrong polarity; it should compare `items[0].substring(4, 6)` against `expectedGrantShortcut` and return false when they differ, otherwise it reads across the token-type/grant-type boundary and rejects matching shortcuts.",
+          "confidence": 0.99,
+          "reasoning": "The candidate identifies the same two problems: the grant shortcut should be extracted with substring(4, 6), and the equality/polarity logic is reversed so matching shortcuts should not be rejected."
+        },
+        {
+          "golden_comment": "Javadoc mentions \"usually like 3-letters shortcut\" but some implementations use 2-letter shortcuts (\"ac\", \"cc\", \"rt\", \"te\", \"pc\", \"ci\", \"ro\"). Consider updating documentation to reflect actual usage pattern.",
+          "severity": "Low",
+          "matched_candidate": "The grant shortcut documentation must specify an exactly two-character shortcut rather than a three-letter shortcut; the token-context decoder expects a six-character prefix composed of three two-character segments, so following the current Javadoc would produce an incompatible grant shortcut.",
+          "confidence": 0.91,
+          "reasoning": "The candidate identifies the same documentation mismatch: the Javadoc suggests three-letter shortcuts while actual usage/decoding relies on two-character shortcuts. It adds implementation detail and is more prescriptive, but the underlying issue is the same."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The lazy cache lookup in `getShortcutByGrantType` uses a non-atomic get/null-check/discovery/put sequence, so concurrent misses for the same grant can repeat `getProviderFactory` calls before either caller populates the `ConcurrentHashMap`."
+        },
+        {
+          "candidate": "The lazy cache lookup in `getGrantTypeByShortcut` uses a non-atomic get/null-check/discovery/put sequence, so concurrent misses for the same shortcut can repeat the full provider stream scan before either caller populates the `ConcurrentHashMap`."
+        },
+        {
+          "candidate": "The newly introduced `TokenContextEncoderProvider` contract methods lack method-level Javadoc defining `rawTokenId` versus `encodedTokenId`, the returned context, and relevant failure behavior, leaving callers and implementations without a specified encoding contract."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": " Catching generic RuntimeException is too broad. The implementation throws IllegalArgumentException specifically - catch that instead for more precise testing.",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 6,
+      "total_scored_candidates": 6,
+      "total_golden": 4,
+      "total_ignored_golden": 0,
+      "tp": 3,
+      "fp": 3,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.75,
+      "tool": "deputydev-v1",
+      "repo_name": "keycloak__keycloak__deputydev-v5__PR37634__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR37634__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/keycloak/keycloak/pull/38446": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "`getCredentials` dereferences the result of `getMyUser(user)` without checking for null; if the user is absent from the internal users map, `getMyUser(user)` returns null and accessing `myUser.recoveryCodes` throws a `NullPointerException` instead of returning an empty credential stream."
+        },
+        {
+          "candidate": "Successful federated recovery-code authentication only checks that the submitted code is present in the stored list and does not remove or persist removal of the matched code, leaving the same recovery code valid for indefinite replay instead of enforcing one-time use."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Unsafe raw List deserialization without type safety. Calling Optional.get() directly on the Optional returned by RecoveryAuthnCodesUtils.getCredential(user) without checking isPresent() can lead to a NoSuchElementException if the Optional is empty.",
+          "severity": "Medium"
+        },
+        {
+          "golden_comment": "After creating the RecoveryAuthnCodesCredentialModel, consider setting its id from the stored credential (e.g., myUser.recoveryCodes.getId()); otherwise getId() will be null and downstream removal by id (e.g., removeStoredCredentialById in the authenticator flow) may not work.",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 2,
+      "total_scored_candidates": 2,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 0,
+      "fp": 2,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "deputydev-v1",
+      "repo_name": "keycloak__keycloak__deputydev-v5__PR38446__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR38446__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/keycloak/keycloak/pull/36882": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "The feature-disabled test for the update metadata invocation asserts only the error text and does not assert exit code 4, so a regression returning a different exit code would still pass even though automation relies on the documented feature-disabled exit-code contract."
+        },
+        {
+          "candidate": "There is no disabled-feature test coverage for the UpdateCompatibilityCheck sub-command; only the metadata sub-command is launched, so removing or breaking the equivalent feature gate in the check sub-command would not be detected."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Incorrect method call for exit codes. The picocli.exit() method calls System.exit() directly, which is problematic:",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 2,
+      "total_scored_candidates": 2,
+      "total_golden": 1,
+      "total_ignored_golden": 0,
+      "tp": 0,
+      "fp": 2,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "deputydev-v1",
+      "repo_name": "keycloak__keycloak__deputydev-v5__PR36882__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR36882__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/keycloak/keycloak/pull/36880": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "When V2 admin permissions are active, the client-removal listener can call `clients().setPermissionsEnabled(client, false)`, which routes through `management(...)` to `MgmtPermissionsV2` and then `ClientPermissionsV2.setPermissionsEnabled`, a method that unconditionally throws `UnsupportedOperationException`; as a result, deleting a client in V2 fails during event processing unless the cleanup is restricted to V1 or implemented with a V2-compatible deletion path."
+        },
+        {
+          "candidate": "The Javadocs for `requireView()` and `requireViewClientScopes()` incorrectly describe boolean return values even though both methods return `void`; their contracts should instead state that they throw `ForbiddenException` when `canView()` or `canViewClientScopes()` returns `false`."
+        },
+        {
+          "candidate": "Existing client management-permissions REST endpoints can select the V2 implementation, but `ClientPermissionsV2.isPermissionsEnabled`, `setPermissionsEnabled`, `resource`, and `getPermissions` all unconditionally throw `UnsupportedOperationException`, so GET or update requests to those public endpoints produce unhandled 500 errors in V2 unless the V1 endpoints are prevented from routing to this implementation or meaningful V2 behavior is provided."
+        },
+        {
+          "candidate": "The private `getEvaluationContext(ClientModel, AccessToken)` method in `ClientPermissionsV2` has no callers while the actual `hasPermission` paths call `root.evaluatePermission(...)` directly, making the method dead code that misleadingly suggests a client-specific evaluation context is used."
+        },
+        {
+          "candidate": "`ClientPermissionsV2` contains an unused static import of `AdminPermissionManagement.TOKEN_EXCHANGE`; the constant is never referenced in the class and the import should be removed."
+        },
+        {
+          "candidate": "Both `hasPermission` overloads in `ClientPermissionsV2` allocate `Arrays.asList(scope)` only to test membership against a single scope value, causing unnecessary allocation on every invocation; they should compare directly with `scope.equals(s)` as `hasGrantedPermission` already does."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleaning up permissions upon role, client, or group removal, is incorrectly guarded by the ADMIN_FINE_GRAINED_AUTHZ (V1) feature flag. This is inconsistent with other methods in the class that use ADMIN_FINE_GRAINED_AUTHZ_V2. Consequently, if ADMIN_FINE_GRAINED_AUTHZ_V2 is enabled but V1 is not, the permission cleanup logic will not execute, leading to orphaned permission data. Cleanup should occur regardless of which fine-grained authorization version is enabled.",
+          "severity": "High"
+        },
+        {
+          "golden_comment": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.getId()), but AdminPermissionsSchema.getOrCreateResource creates per-client resources with the owner set to resourceServer.getClientId(), so this lookup will never find those resources and will always fall back to the 'all-clients' resource, effectively ignoring client-specific permissions.",
+          "severity": "High"
+        },
+        {
+          "golden_comment": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RESOURCE_TYPE) and returning resource.getName() will only ever consider the type-level 'Clients' resource (per-client resources have no type) and return its name, while AvailableRoleMappingResource#getRoleIdsWithPermissions expects actual client IDs to pass to realm.getClientById, which can lead to incorrect behavior or a null client and subsequent failures.",
+          "severity": "High"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 6,
+      "total_scored_candidates": 6,
+      "total_golden": 3,
+      "total_ignored_golden": 0,
+      "tp": 0,
+      "fp": 6,
+      "fn": 3,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "deputydev-v1",
+      "repo_name": "keycloak__keycloak__deputydev-v5__PR36880__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR36880__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/keycloak/keycloak/pull/37038": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Incorrect permission check in canManage() method",
+          "severity": "High",
+          "matched_candidate": "`GroupPermissionsV2.canManage()` passes both `VIEW` and `MANAGE` scopes to `hasPermission`, whose loop succeeds if any requested scope is granted, so a caller with only all-groups `VIEW` permission can pass a manage authorization gate; the check should require `AdminPermissionsSchema.MANAGE` only.",
+          "confidence": 0.98,
+          "reasoning": "The candidate specifically identifies that canManage() incorrectly checks both VIEW and MANAGE permissions, allowing VIEW-only users to pass a manage check, which is the same underlying permission-check bug described by the golden comment."
+        },
+        {
+          "golden_comment": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() and the same groupResource.getId() is added to granted, but hasPermission resolves resources by name (treating the argument as a group id) and the GroupPermissionEvaluator contract says this method returns group IDs that are later used as UserModel.GROUPS and in getUsersCount group filters. This mismatch means per-group VIEW_MEMBERS/MANAGE_MEMBERS permissions may not yield the expected group IDs for filtering and counts, and evaluation may effectively only look at the type-level 'all-groups' resource; consider revisiting whether this should operate on the underlying group ids (resource names) instead so it aligns with the JPA queries and the interface contract.",
+          "severity": "High",
+          "matched_candidate": "The callback passes `groupResource.getId()` to `hasPermission`, but that value is the authorization resource UUID rather than the group model ID stored in `groupResource.getName()`, causing `findByName` to miss the group-specific permission resource and fall back to all-groups evaluation.",
+          "confidence": 0.95,
+          "reasoning": "The candidate identifies the same core mismatch: using the authorization resource UUID (`groupResource.getId()`) where the group model ID/resource name is expected, causing `hasPermission`/`findByName` to miss group-specific permissions and rely on all-groups behavior. It omits some downstream filtering/count contract details but matches the underlying bug."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The Javadoc for `requireManageMembers(GroupModel)` is copy-pasted from `requireManageMembership` and incorrectly references `canManageMembership(GroupModel)` instead of the semantically correct `canManageMembers(GroupModel)`, misleading API consumers and implementors about which permission check the method enforces."
+        },
+        {
+          "candidate": "The Javadoc for `getGroupIdsWithViewPermission()` describes the return value as a stream even though the method signature returns `Set<String>`, giving callers an incorrect API contract instead of documenting a set of group IDs."
+        },
+        {
+          "candidate": "The callback adds `groupResource.getId()` to the returned `granted` set, but that value is the authorization resource UUID rather than the group model ID stored in `groupResource.getName()`, so provider-side user filtering receives invalid group IDs."
+        }
+      ],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 5,
+      "total_scored_candidates": 5,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 3,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.4,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "keycloak__keycloak__deputydev-v5__PR37038__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR37038__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/keycloak/keycloak/pull/33832": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Returns wrong provider (default keystore instead of BouncyCastle)",
+          "severity": "High",
+          "matched_candidate": "AuthzClientCryptoProvider.getBouncyCastleProvider() returns the provider backing the JDK default keystore type instead of an actual BouncyCastle provider or an explicit unsupported signal, so callers expecting BouncyCastle can receive an unrelated JDK keystore provider and fail on provider-specific algorithm lookups.",
+          "confidence": 0.99,
+          "reasoning": "The candidate explicitly states that getBouncyCastleProvider() returns the provider for the JDK default keystore rather than an actual BouncyCastle provider, matching the golden issue of returning the wrong provider."
+        },
+        {
+          "golden_comment": "Dead code exists where ASN1Encoder instances are created and written to, but their results are immediately discarded. The actual encoding is performed by new ASN1Encoder instances created in the subsequent return statement, rendering the earlier operations useless.",
+          "severity": "Low",
+          "matched_candidate": "The two standalone ASN1Encoder.create().write(...) calls in AuthzClientCryptoProvider discard their results and are immediately repeated in the return expression, so they have no effect on the output and cause redundant allocations and DER encoding work on every invocation.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same dead/redundant code: ASN1Encoder instances are created and written to, their results are discarded, and the work is repeated in the return expression. This matches the golden issue precisely."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The added Hamcrest test dependency in authz/client/pom.xml is unused because the module's tests use org.junit.Assert.assertArrayEquals and have no Hamcrest imports or matcher usage, so it adds unnecessary build configuration."
+        },
+        {
+          "candidate": "ASN1Decoder.readLength() returns -1 for indefinite-length encodings, but the code enters a while (length > 0) path that skips parsing and returns an empty sequence instead of rejecting the unsupported DER encoding, causing malformed indefinite-length input to fail later with a misleading sequence-size error."
+        }
+      ],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 4,
+      "total_scored_candidates": 4,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 2,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "keycloak__keycloak__deputydev-v5__PR33832__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR33832__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/keycloak/keycloak/pull/40940": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Returning null from getSubGroupsCount() violates the GroupModel contract (Javadoc says it never returns null) and may lead to NPEs in callers that expect a non-null count.",
+          "severity": "Critical",
+          "matched_candidate": "GroupAdapter.getSubGroupsCount() returns null when modelSupplier.get() is absent, violating the GroupModel.getSubGroupsCount() contract that it never returns null and allowing null propagation to representations or unboxing failures in callers; it should return a non-null fallback such as 0L.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same issue: getSubGroupsCount() can return null when the model is absent, violating the GroupModel contract that the method should never return null and potentially causing NPEs/unboxing failures. It also suggests a non-null fallback like 0L, consistent with the golden concern."
+        },
+        {
+          "golden_comment": "The reader thread isn’t waited for; flipping deletedAll to true and asserting immediately can race and miss exceptions added just after the flag change, making this test flaky.",
+          "severity": "Medium",
+          "matched_candidate": "The test checks caughtExceptions without retaining and joining the reader thread, so the reader may still be inside groups() and append an exception after the assertion has already passed, causing false passes.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same race: the reader thread is not joined/waited for before asserting, so it can add exceptions after the assertion, making the test unreliable/flaky."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The adjacent getSubGroupsStream(...) overloads still directly dereference modelSupplier.get(), so if the supplied model is null during concurrent deletion they can throw an NPE; they should guard the null model case and return an empty stream."
+        },
+        {
+          "candidate": "The test does not synchronize or record that at least one reader call completed during the deletion loop, so the deletions can finish without actually exercising the concurrent-read regression path."
+        }
+      ],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 4,
+      "total_scored_candidates": 4,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 2,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "keycloak__keycloak__deputydev-v5__PR40940__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR40940__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "ConditionalPasskeysEnabled() called without UserModel parameter",
+          "severity": "Medium",
+          "matched_candidate": "In `UsernameForm.java`, the newly added zero-argument call to `isConditionalPasskeysEnabled()` does not compile because the available inherited method requires a `UserModel` argument; pass the already-checked user, e.g. `isConditionalPasskeysEnabled(context.getUser())`, when `context.getUser() != null`.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same issue: a zero-argument call to `isConditionalPasskeysEnabled()`/`ConditionalPasskeysEnabled()` is incorrect because the method requires a `UserModel` parameter, and suggests passing `context.getUser()`."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "With isConditionalPasskeysEnabled(UserModel user) requiring user != null, authenticate(...) will not call webauthnAuth.fillContextForm(context) on the initial login page where context.getUser() is still null, so conditional passkey UI will not be set up for first-time passkey login. Consider whether this should also be enabled when no user has been selected yet so ID-less passkey authentication on the initial login form continues to work.",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 1,
+      "total_scored_candidates": 1,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 0,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.5,
+      "tool": "deputydev-v1",
+      "repo_name": "keycloak__keycloak-greptile__deputydev-v5__PR1__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak-greptile__deputydev-v5__PR1__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/getsentry/sentry/pull/93824": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Inconsistent metric tagging with 'shard' and 'shards'",
+          "severity": "Medium",
+          "matched_candidate": "The `spans.buffer.flusher.wait_produce` timer uses the tag key `shards` while neighboring metrics use `shard` for the same `shard_tag` value, creating an inconsistent metric dimension that causes queries and dashboards filtering on `shard` to omit this timer.",
+          "confidence": 1.0,
+          "reasoning": "The candidate explicitly identifies the inconsistent metric tag key usage between `shards` and `shard` for the same shard value, which is exactly the issue described by the golden comment."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The restart limit guard uses `>` before incrementing `self.process_restarts[process_index]`, so a worker whose restart count already equals `MAX_PROCESS_RESTARTS` is restarted one extra time and the RuntimeError is only raised after the following failure; the check should use `>= MAX_PROCESS_RESTARTS`."
+        },
+        {
+          "candidate": "The `SpanFlusher` docstring incorrectly says it creates one process per shard, but `num_processes` is capped by `max_processes` and the round-robin mapping can assign multiple shards to each worker, so the documentation contradicts the implemented capped shared-shard worker model."
+        },
+        {
+          "candidate": "Negative `max_processes` values are accepted and remain truthy, causing `num_processes` to become negative and leaving no valid process entries for shard assignment; the constructor should reject `max_processes < 1` before building the worker-to-shard mapping."
+        },
+        {
+          "candidate": "When `produce_to_pipe` is set, the worker is a `threading.Thread`, but the restart cleanup only handles multiprocessing workers before starting a replacement, so a hung thread can remain alive while another worker is started for the same shards across repeated restarts."
+        },
+        {
+          "candidate": "The comment describes an individual-shard health check, but the code reads a single `process_healthy_since[process_index]` timestamp and marks the entire worker as hung, so the comment should describe a process-level hang check instead of shard-specific health evaluation."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Fixed sleep in tests can be flaky; wait on condition instead",
+          "severity": "Low"
+        },
+        {
+          "golden_comment": "Because flusher processes are created via multiprocessing.get_context('spawn').Process, they are instances of multiprocessing.context.SpawnProcess, which on POSIX is not a subclass of multiprocessing.Process, so this isinstance check will always be false and hung processes won't be killed here.",
+          "severity": "High"
+        },
+        {
+          "golden_comment": "Sleep in test_consumer.py won’t actually wait because time.sleep was monkeypatched above; consider restoring sleep or using a different sync to ensure the flusher has time to process.",
+          "severity": "Medium"
+        },
+        {
+          "golden_comment": "Breaking out of the loop when the deadline has elapsed can skip terminating remaining flusher processes, potentially leaving them running after shutdown; consider ensuring termination is attempted even if the deadline is exceeded.",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 6,
+      "total_scored_candidates": 6,
+      "total_golden": 5,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 5,
+      "fn": 4,
+      "errors_count": 0,
+      "precision": 0.16666666666666666,
+      "recall": 0.2,
+      "tool": "deputydev-v1",
+      "repo_name": "sentry__sentry__deputydev-v5__PR93824__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/sentry__sentry__deputydev-v5__PR93824__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).",
+          "severity": "Low",
+          "matched_candidate": "The code pairs error_ids with events.values() by position even though nodestore get_multi may return a dictionary in a different order or omit entries, which can attach an event's title, message, and timestamp to the wrong error ID; returned events should be associated by node_id using a node_id-to-event_id mapping.",
+          "confidence": 0.99,
+          "reasoning": "The candidate identifies the same issue: pairing error_ids with events.values() by position relies on get_multi dictionary ordering and presence of all entries, which can misassociate event data with the wrong error ID. It also notes the correct approach of mapping by node_id."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The age/timestamp mutual-exclusivity validation uses truthiness checks instead of presence checks, so zero-valued fields such as age=0 with timestamp or timestamp=0 with age can bypass validation and allow both mutually exclusive fields in the same payload."
+        },
+        {
+          "candidate": "When the use-table-widget-visualization feature is enabled, the code passes empty columns, data, fields, and units to TableWidgetVisualization instead of using result.data and result.meta, causing organizations with the feature enabled to render a blank table regardless of available query results."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Breaking changes in error response format",
+          "severity": "Medium"
+        },
+        {
+          "golden_comment": "Detector validator uses wrong key when updating type",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 3,
+      "total_scored_candidates": 3,
+      "total_golden": 3,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 2,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.3333333333333333,
+      "recall": 0.3333333333333333,
+      "tool": "deputydev-v1",
+      "repo_name": "sentry__sentry-greptile__deputydev-v5__PR5__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/sentry__sentry-greptile__deputydev-v5__PR5__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Django querysets do not support negative slicing",
+          "severity": "High",
+          "matched_candidate": "The optimized paginator branch for `cursor.offset < 0` assigns the negative offset to `start_offset` and uses it as the start of a Django QuerySet slice, but Django rejects negative QuerySet indices, causing optimized requests with such cursors to raise an unhandled exception instead of returning a page.",
+          "confidence": 1.0,
+          "reasoning": "The candidate explicitly identifies that a negative offset is used in a Django QuerySet slice and that Django rejects negative QuerySet indices, which is the same underlying issue as the golden comment."
+        },
+        {
+          "golden_comment": "get_item_key assumes a numeric key, but the paginator is used with order_by=-datetime in the audit logs endpoint; calling math.floor/ceil on a datetime will raise a TypeError.",
+          "severity": "High",
+          "matched_candidate": "`OptimizedCursorPaginator.get_item_key` passes the ordering value directly to `math.floor` or `math.ceil`, but the audit-log endpoint configures it with `order_by=\"-datetime\"`, so the value is a `datetime` and optimized cursor generation raises `TypeError` on every response unless datetime keys are converted to the numeric representation used by the existing datetime paginator.",
+          "confidence": 1.0,
+          "reasoning": "The candidate describes the same bug: get_item_key uses math.floor/ceil on the ordering value, but the audit-log paginator orders by a datetime field, causing a TypeError unless converted to a numeric representation."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The audit-log endpoint has two branches that duplicate the same `self.paginate()` arguments and differ only in `paginator_cls` and `enable_advanced_features`, creating two locations where shared request, queryset, ordering, and serialization arguments can diverge during future changes."
+        },
+        {
+          "candidate": "The paginator comment incorrectly states that negative offsets are safe because the underlying queryset handles boundary conditions, but Django QuerySets reject negative slice indices, so the documentation should remove that safety claim or document the required non-negative offset invariant."
+        },
+        {
+          "candidate": "`OptimizedCursorPaginator.get_result` duplicates the base paginator's cursor initialization, hit counting, boundary trimming, reversal, cursor construction, and post-query filtering even though only offset selection differs, so fixes or cursor-handling changes made to the base pagination flow may not reach the optimized paginator and the implementations can diverge."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Importing non-existent OptimizedCursorPaginator",
+          "severity": "Low"
+        },
+        {
+          "golden_comment": "When requests are authenticated with API keys or org auth tokens (which have user_id=None), organization_context.member is None. Line 71 attempts to access organization_context.member.has_global_access without checking if member is None, causing an AttributeError crash when optimized_pagination=true is used, even though the request passed all permission checks with valid org:write scope.",
+          "severity": "High"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 5,
+      "total_scored_candidates": 5,
+      "total_golden": 4,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 3,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.4,
+      "recall": 0.5,
+      "tool": "deputydev-v1",
+      "repo_name": "sentry__sentry-greptile__deputydev-v5__PR1__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/sentry__sentry-greptile__deputydev-v5__PR1__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/grafana/grafana/pull/97529": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "In `NewResourceServer`, logging the error returned by `s.Init(ctx)` is redundant because `server.Init` already logs `s.log.Error(\"error initializing resource server\", \"error\", s.initErr)` before returning the same failure, causing two identical \"error initializing resource server\" log lines for a single initialization failure."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "A race condition in BuildIndex allows multiple goroutines to concurrently build the same expensive index for the same key. This is caused by moving the b.cacheMu lock from protecting the entire function to only protecting the final cache assignment. ",
+          "severity": "High"
+        },
+        {
+          "golden_comment": "Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchronization, and the event watcher goroutine started just above could trigger BuildIndex writes concurrently, potentially causing a concurrent map read/write panic.",
+          "severity": "High"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 1,
+      "total_scored_candidates": 1,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 0,
+      "fp": 1,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "deputydev-v1",
+      "repo_name": "grafana__grafana__deputydev-v5__PR97529__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR97529__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/getsentry/sentry/pull/80168": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "MetricAlertDetectorHandler inherits from StatefulDetectorHandler but only contains pass, failing to implement its required abstract methods: counter_names (property), get_dedupe_value(), get_group_key_values(), and build_occurrence_and_event_data(). This will cause a TypeError at runtime when the class is instantiated.",
+          "severity": "High",
+          "matched_candidate": "`MetricAlertDetectorHandler` now extends `StatefulDetectorHandler` without implementing its abstract hooks, including `build_occurrence_and_event_data`, so Python treats the handler as abstract and raises `TypeError` when it is instantiated, blocking any path that resolves this detector handler.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same issue: MetricAlertDetectorHandler inherits from StatefulDetectorHandler but does not implement required abstract methods/hooks, causing Python to treat it as abstract and raise TypeError on instantiation. It mentions one hook explicitly but clearly refers to the same underlying bug."
+        },
+        {
+          "golden_comment": "Docstring says this returns a list of DetectorEvaluationResult, but the method now returns a dict keyed by DetectorGroupKey. Consider updating the docstring to match the new return type.",
+          "severity": "Low",
+          "matched_candidate": "The `evaluate` docstring still says the method returns a list even though its annotation and implementation now return a `dict[DetectorGroupKey, DetectorEvaluationResult]`, causing the documentation to directly contradict the code.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same issue: the evaluate docstring incorrectly says it returns a list while the code/annotation now return a dict keyed by DetectorGroupKey."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 2,
+      "total_scored_candidates": 2,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 0,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "sentry__sentry__deputydev-v5__PR80168__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/sentry__sentry__deputydev-v5__PR80168__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/getsentry/sentry/pull/80528": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The function modifies the config variable to include display values but then returns the original monitor.config instead of the modified version.",
+          "severity": "High",
+          "matched_candidate": "The function copies the monitor config and replaces `schedule_type` with its display value, but returns `monitor_environment.monitor.config` instead of the local transformed `config`, discarding the transformation and exposing the original `schedule_type` value in the returned monitor context.",
+          "confidence": 1.0,
+          "reasoning": "The candidate describes the same bug: a local copied/modified config with display values is created, but the function returns the original monitor config instead, so the transformation is lost."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "`create_incident_occurrence` is called for every check-in in `incidents.py`, causing `monitor_env.get_last_successful_checkin()` to be fetched repeatedly for the same monitor environment; incidents containing multiple check-ins perform the same last-successful-check-in lookup once per occurrence instead of once per monitor environment, so the value should be fetched once before the loop and passed in."
+        },
+        {
+          "candidate": "`get_human_failure_reason` has no branch for a zero total, so when the input is empty or every status is excluded by `HUMAN_FAILURE_STATUS_MAP`, `get_text_list([])` produces an empty value and the function returns the malformed user-facing string `\" check-ins detected\"` instead of the documented fallback."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The code fetches MonitorCheckIn objects by ID when the required data already exists in previous_checkins. This creates an unnecessary database query.",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 3,
+      "total_scored_candidates": 3,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 2,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.3333333333333333,
+      "recall": 0.5,
+      "tool": "deputydev-v1",
+      "repo_name": "sentry__sentry__deputydev-v5__PR80528__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/sentry__sentry__deputydev-v5__PR80528__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/getsentry/sentry/pull/77754": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Shared mutable default in dataclass timestamp",
+          "severity": "Medium",
+          "matched_candidate": "`timezone.now()` is evaluated once when `AssignmentSource` is defined, so every instance that omits `queued` receives the same stale module-import timestamp instead of a fresh timestamp; use `field(default_factory=timezone.now)` for the `queued` dataclass field.",
+          "confidence": 0.98,
+          "reasoning": "The candidate points out that the dataclass timestamp default is evaluated once at class definition and shared across instances, recommending default_factory, which matches the golden issue."
+        },
+        {
+          "golden_comment": "The method name has a typo: test_from_dict_inalid_data should be test_from_dict_invalid_data.",
+          "severity": "Low",
+          "matched_candidate": "The test name `test_from_dict_inalid_data` contains a typo; rename it to `test_from_dict_invalid_data` so the identifier is readable and appears correctly in test reports.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same typo in the test method name and suggests the same corrected spelling, `test_from_dict_invalid_data`."
+        },
+        {
+          "golden_comment": "Method name says 'empty_array' but tests empty dict - consider renaming to 'test_from_dict_empty_dict' for clarity.",
+          "severity": "Low",
+          "matched_candidate": "The test name incorrectly describes its input as an array even though it passes an empty dictionary; rename it to `test_from_dict_empty_dict` so the test name accurately reflects the tested input.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same mismatch between the test method name saying array and the actual empty dictionary input, and suggests the same rename for clarity."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "to_dict() returns a datetime for queued; if this dict is passed in task kwargs (e.g., via apply_async), JSON serialization may fail depending on the serializer, which can cause enqueue errors.",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 3,
+      "total_scored_candidates": 3,
+      "total_golden": 4,
+      "total_ignored_golden": 0,
+      "tp": 3,
+      "fp": 0,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.75,
+      "tool": "deputydev-v1",
+      "repo_name": "sentry__sentry__deputydev-v5__PR77754__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/sentry__sentry__deputydev-v5__PR77754__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/getsentry/sentry/pull/95633": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "The worker unconditionally marks an offset complete in the finally block even when result_processor raises and the exception is swallowed, making the failed message eligible for commit and preventing redelivery despite the strategy's guarantee that offsets are committed only after successful processing."
+        },
+        {
+          "candidate": "The work queue is unbounded, so if producers submit messages faster than workers process them, queued WorkItems retain full messages and can cause unbounded heap growth instead of applying backpressure."
+        },
+        {
+          "candidate": "shutdown sets each worker's shutdown flag before gracefully shutting down the queues, so workers exit after their current item instead of draining the queued backlog, leaving remaining offsets outstanding and defeating shutdown(immediate=False)."
+        },
+        {
+          "candidate": "The exception log for queue worker processing failures includes only the worker ID and omits the available partition and offset, making it impossible to identify which Kafka message failed from the log context."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The queue.shutdown() method with 'immediate=False' parameter may not exist in the standard Python queue module. This could cause AttributeError at runtime. Verify the correct API or implement a custom shutdown mechanism.",
+          "severity": "High"
+        },
+        {
+          "golden_comment": "The magic number 50 for max_wait is used repeatedly throughout the tests. Consider extracting this as a named constant to improve maintainability.",
+          "severity": "Low"
+        },
+        {
+          "golden_comment": "The test test_thread_queue_parallel_error_handling has a docstring that doesn't match the test implementation.",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 4,
+      "total_scored_candidates": 4,
+      "total_golden": 3,
+      "total_ignored_golden": 0,
+      "tp": 0,
+      "fp": 4,
+      "fn": 3,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "deputydev-v1",
+      "repo_name": "sentry__sentry__deputydev-v5__PR95633__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/sentry__sentry__deputydev-v5__PR95633__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "OptimizedCursorPaginator negative-offset branch slices QuerySet with a negative start index",
+          "severity": "Critical",
+          "matched_candidate": "With advanced pagination enabled, a negative previous-page cursor offset is assigned directly to `start_offset` and used in `queryset[start_offset:stop]`; Django querysets do not support negative slice indices, so such requests will raise at runtime instead of paginating unless the offset is clamped or the branch is removed.",
+          "confidence": 0.98,
+          "reasoning": "The candidate describes the same issue: in the advanced/OptimizedCursorPaginator path, a negative previous-page cursor offset is used as the start index in a QuerySet slice, which Django does not allow and will raise at runtime."
+        },
+        {
+          "golden_comment": "BasePaginator negative-offset branch slices QuerySet with a negative start index",
+          "severity": "High",
+          "matched_candidate": "With advanced pagination enabled, a negative previous-page cursor offset is assigned directly to `start_offset` and used in `queryset[start_offset:stop]`; Django querysets do not support negative slice indices, so such requests will raise at runtime instead of paginating unless the offset is clamped or the branch is removed.",
+          "confidence": 0.98,
+          "reasoning": "The candidate describes the same issue: a negative offset is used as the start index when slicing a Django QuerySet, which QuerySets do not support and can raise at runtime. This matches the golden comment about BasePaginator's negative-offset branch slicing with a negative start index."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "`OptimizedCursorPaginator.get_result` duplicates the base paginator flow for cursor initialization, hit counting, slicing, previous-page handling, cursor construction, and post-query filtering, so fixes or changes to the base implementation can silently diverge; factor the shared flow into the inheritance path and override only the differing offset behavior."
+        },
+        {
+          "candidate": "The comment claims negative offsets are safe because the queryset handles boundary conditions, but the code clamps non-previous offsets to avoid invalid negative queryset slicing; update the documentation to describe the actual clamping behavior instead of misrepresenting queryset negative-index support."
+        },
+        {
+          "candidate": "The Lua buffering script now applies sorted-set commands such as `zcard` and `zunionstore` to keys that previously held Redis Sets, so existing keys or keys written by old instances during a rolling deployment can trigger `WRONGTYPE` and abort the script; add a migration or use a new key namespace to prevent Set and Sorted Set data from overlapping."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "OptimizedCursorPaginator.get_item_key uses floor/ceil on a datetime key (order_by='-datetime'), causing TypeError.",
+          "severity": "High"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 4,
+      "total_scored_candidates": 5,
+      "total_golden": 3,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 3,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.4,
+      "recall": 0.6666666666666666,
+      "tool": "deputydev-v1",
+      "repo_name": "sentry__sentry-greptile__deputydev-v5__PR2__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/sentry__sentry-greptile__deputydev-v5__PR2__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "sample_rate = 0.0 is falsy and skipped",
+          "severity": "Low",
+          "matched_candidate": "The factory checks `client_sample_rate` by truthiness before assigning `sample_rate`, so a numeric `client_sample_rate` of `0.0` is treated as absent and skipped even though it is present and convertible; check explicitly for `None` before converting to preserve the zero sample-rate edge case.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same issue: a sample rate value of 0.0 is falsy, so the truthiness check incorrectly skips assigning/converting it instead of treating it as a valid present value."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The comment claims aliasing `should_upsample` as `upsampling_enabled` provides query optimization and caching, but the assignment only renames the eligibility result and adds no optimization or caching behavior; remove the unsupported claim and either use `should_upsample` directly or assign the eligibility result directly to `upsampling_enabled`."
+        },
+        {
+          "candidate": "Enabling the `upsampled_count()` rewrite for the errors dataset can emit `upsampled_count() as count` for an errors-dataset request containing `count()`, but `upsampled_count` is only registered in Discover and not in the errors dataset function converter, so the errors query builder cannot resolve the generated function; register `upsampled_count` for the errors dataset or keep this branch disabled until that registration exists."
+        },
+        {
+          "candidate": "Using a raw substring check for `event.type:error` falsely matches negated, quoted, or mixed event-type queries such as `!event.type:error`, `NOT event.type:error`, or quoted text containing that string, causing the error-only upsampled aggregation transformation to be applied to queries that exclude errors or include non-error events; replace it with query classification that confirms a positive `event.type:error` filter and rejects negated, quoted, or mixed-type cases."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Using Python’s built-in hash() to build cache keys is non-deterministic across processes (hash randomization), so keys won’t match across workers and invalidate_upsampling_cache may fail to delete them. Use a deterministic serialization of project_ids for the cache key.",
+          "severity": "Low"
+        },
+        {
+          "golden_comment": "The upsampling eligibility check passes the outer dataset instead of the actual dataset used by scoped_dataset. In paths where the query ultimately runs against discover (e.g., dashboard split) while the original dataset is metrics, upsampling may be skipped even when all projects are allowlisted.",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 4,
+      "total_scored_candidates": 4,
+      "total_golden": 3,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 3,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.25,
+      "recall": 0.3333333333333333,
+      "tool": "deputydev-v1",
+      "repo_name": "sentry__sentry-greptile__deputydev-v5__PR3__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/sentry__sentry-greptile__deputydev-v5__PR3__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/grafana/grafana/pull/103633": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "A cached denial in `permDenialCache` is returned before consulting the positive permission cache or database, so after a permission is later granted, the stale denial can continue returning `Allowed: false` until `shortCacheTTL` expires; this also affects action-level checks whose denial key has empty name and parent segments."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The Check operation exhibits asymmetric cache trust logic: cached permission grants are trusted and returned immediately, but cached denials from the same permission cache are ignored, leading to a fresh database lookup. This allows stale cached grants to provide access to revoked resources, posing a security risk. ",
+          "severity": "High"
+        },
+        {
+          "golden_comment": "The test comment says the cached permissions 'allow access', but the map stores false for dashboards:uid:dash1, so checkPermission will still treat this scope as not allowed.",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 1,
+      "total_scored_candidates": 1,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 0,
+      "fp": 1,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "deputydev-v1",
+      "repo_name": "grafana__grafana__deputydev-v5__PR103633__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR103633__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/getsentry/sentry/pull/67876": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The code attempts to access integration.metadata[sender][login] without checking for the existence of the sender key. This causes a KeyError for integrations where the sender metadata was not set during creation",
+          "severity": "High",
+          "matched_candidate": "The code directly accesses nested `integration.metadata[\"sender\"][\"login\"]` even though `sender` is only conditionally populated, so an existing integration missing `sender` or `login` raises `KeyError` instead of returning the controlled validation error.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same underlying bug: direct nested access to integration.metadata[\"sender\"][\"login\"] when the sender metadata may be absent, causing a KeyError for integrations without sender metadata."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The call to `get_user_info(payload[\"access_token\"])` is outside any exception handler even though `get_user_info` raises for non-2xx GitHub responses via `raise_for_status`, so invalid tokens, rate limits, or GitHub failures can escape `OAuthLoginView.dispatch` as an unhandled 500 instead of showing the intended installation error page."
+        },
+        {
+          "candidate": "The OAuth token validation only checks whether the `access_token` key exists, so a parsed response containing an empty `access_token=` passes the condition and sends an empty credential to `get_user_info`; the code should validate that the token value is non-empty."
+        },
+        {
+          "candidate": "The new synchronous `get_user_info` request uses an underlying `session.get` without an explicit timeout, so an unresponsive or degraded GitHub API can hold the installation request worker indefinitely."
+        },
+        {
+          "candidate": "The same `Integration` row is queried twice on the path where an integration exists without an `OrganizationIntegration`; the code should retain and reuse the `Integration` loaded for the organization-membership check and inspect its `status` to avoid the extra database round trip."
+        },
+        {
+          "candidate": "`safe_urlopen` is called before the surrounding `try` block, so connection, TLS, timeout, or other network failures while opening the OAuth token endpoint escape as a 500 instead of being converted into the controlled installation error."
+        },
+        {
+          "candidate": "When `Integration.DoesNotExist` is hit, the code advances a new installation before verifying that `github_authenticated_user` matches the GitHub user who installed the app, allowing a supplied installation ID with no existing record to bypass the only authenticated-user versus sender identity check."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Null reference if github_authenticated_user state is missing",
+          "severity": "Medium"
+        },
+        {
+          "golden_comment": "OAuth state uses pipeline.signature (static) instead of a per-request random value",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 7,
+      "total_scored_candidates": 7,
+      "total_golden": 3,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 6,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.14285714285714285,
+      "recall": 0.3333333333333333,
+      "tool": "deputydev-v1",
+      "repo_name": "sentry__sentry__deputydev-v5__PR67876__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/sentry__sentry__deputydev-v5__PR67876__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/keycloak/keycloak/pull/32918": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Cleanup reference uses incorrect alias - should be 'idp-alias-' + i instead of 'alias'.",
+          "severity": "Medium",
+          "matched_candidate": "Cleanup is registered with the literal alias \"alias\" instead of the actual IDP aliases created by the test (\"idp-alias-\" + i and \"idp-alias-20\"), so the cleanup callbacks target non-existent providers and leave the created identity providers in the test realm, potentially contaminating subsequent tests.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same underlying bug: cleanup callbacks use the literal string \"alias\" rather than the actual generated IDP alias such as \"idp-alias-\" + i, causing cleanup to target the wrong providers."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "Recursive caching call using session instead of delegate",
+          "severity": "Critical"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 1,
+      "total_scored_candidates": 1,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 0,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.5,
+      "tool": "deputydev-v1",
+      "repo_name": "keycloak__keycloak__deputydev-v5__PR32918__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/keycloak__keycloak__deputydev-v5__PR32918__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/grafana/grafana/pull/94942": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The enableSqlExpressions function has flawed logic that always returns false, effectively disabling SQL expressions unconditionally:",
+          "severity": "Critical",
+          "matched_candidate": "`enableSqlExpressions` performs a feature-flag lookup but returns `false` on both paths, making the lookup dead code and misleadingly suggesting the SQL-expression security kill switch can be enabled; this obscures that the permanent disablement is intentional and increases the risk that a future maintainer accidentally re-enables the path.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same core issue: `enableSqlExpressions` does a feature-flag lookup but returns `false` regardless, so SQL expressions are always disabled and the lookup is effectively dead/misleading code."
+        },
+        {
+          "golden_comment": "Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto return 'not implemented'.",
+          "severity": "High",
+          "matched_candidate": "`pkg/expr/sql` exposes a `DB` API whose operations are only failing `not implemented` stubs, but the package lacks documentation explaining that this is an intentional disabled stub and security boundary; without that context, the code looks like unfinished scaffolding and makes accidental later implementation more likely.",
+          "confidence": 0.9,
+          "reasoning": "The candidate explicitly identifies that the DB API operations are `not implemented` stubs, which is the same underlying issue as the golden comment noting methods like `RunCommands` and `QueryFramesInto` return `not implemented`. The additional documentation/security-boundary framing does not change the core match."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 2,
+      "total_scored_candidates": 2,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 0,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "grafana__grafana__deputydev-v5__PR94942__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR94942__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/grafana/grafana/pull/90939": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The GetWebAssets function implements an incomplete double-checked locking pattern for caching web assets. The function first checks if the cache is populated using a read lock (RLock), and if the cache is empty, it acquires a write lock to populate it. However, it fails to re-check whether the cache was populated by another goroutine while waiting to acquire the write lock.",
+          "severity": "Medium",
+          "matched_candidate": "In non-Dev environments, callers that observe `entryPointAssetsCache == nil` under the read lock do not re-check the cache after acquiring the write lock, so multiple concurrent callers can serially repeat the expensive filesystem or CDN initialization even after the first caller has populated the cache, overwriting the cached result.",
+          "confidence": 0.98,
+          "reasoning": "The candidate describes the same incomplete double-checked locking bug: after seeing a nil cache under a read lock, callers acquire the write lock but fail to re-check whether another goroutine already populated the cache, causing repeated initialization and overwrites."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "In addition to the missing double-check, the function has a critical flaw in its error handling: it unconditionally assigns the fetch result to the cache (line 69: entryPointAssetsCache = result) regardless of whether the fetch succeeded or failed. When an error occurs during asset fetching, result is nil, and this nil value overwrites any previously valid cache entry.",
+          "severity": "High"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 1,
+      "total_scored_candidates": 1,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 0,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.5,
+      "tool": "deputydev-v1",
+      "repo_name": "grafana__grafana__deputydev-v5__PR90939__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR90939__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/grafana/grafana/pull/80329": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The code uses Error log level for what appears to be debugging information. This will pollute error logs in production. Consider using Debug or Info level instead.",
+          "severity": "Low",
+          "matched_candidate": "In xorm_store.go cleanup diagnostics, successful cleanup batches are logged with Error-level calls and pre-delete logs include the complete ids slice and raw condition; configured batches can contain tens of thousands of IDs, causing oversized false error events, log noise, and exposure of row IDs and query details even when no error occurred.",
+          "confidence": 0.95,
+          "reasoning": "The candidate identifies the same core issue: non-error cleanup/debug diagnostic information is being logged at Error level, causing false error log noise in production. It adds extra concerns about payload size and exposure, but the underlying bug matches the golden comment."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "In cleanup_test.go, the subtest registers t.Cleanup only after creating test annotations and running initial assertions; because the subtests share fakeSQL, if createTestAnnotations or an initial assertion calls t.Fatal before cleanup is registered, teardown is skipped and leftover annotation rows can pollute subsequent subtests."
+        },
+        {
+          "candidate": "In xorm_store.go, the SQLite delete path rebuilds the complete values string with fmt.Sprintf on every iteration, repeatedly copying the growing ID list; for large batches such as 32,767 IDs this causes O(n²) copying and allocations, hurting performance."
+        },
+        {
+          "candidate": "In cleanup.go, changing the shared cleanup ticker from every 10 minutes to every minute makes every cleanup job—not only annotation cleanup—eligible to run up to ten times as often, increasing database work across snapshots, dashboard versions, images, annotations, and other cleanup jobs."
+        }
+      ],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 4,
+      "total_scored_candidates": 4,
+      "total_golden": 1,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 3,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.25,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "grafana__grafana__deputydev-v5__PR80329__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR80329__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/grafana/grafana/pull/90045": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The context is being created with d.Log instead of the log variable that was initialized with additional context values (name, kind, method). This means those values won't be propagated to the logging context.",
+          "severity": "Medium",
+          "matched_candidate": "Delete enriches a log variable with name, kind, and method fields but then stores the base d.Log in the context, so downstream logging that retrieves its logger from ctx loses those fields; the enriched log should be placed into the context.",
+          "confidence": 1.0,
+          "reasoning": "The candidate describes the same issue: an enriched logger with name/kind/method is created, but the un enriched base d.Log is stored in the context, so those fields are not propagated downstream."
+        },
+        {
+          "golden_comment": "Bug: calling recordLegacyDuration when storage operation fails should be recordStorageDuration.",
+          "severity": "High",
+          "matched_candidate": "The d.Storage.Update error path uses startStorage but records the failure through recordLegacyDuration, misattributing storage failures and latency to the legacy histogram; it should use recordStorageDuration.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same bug: the storage update error path incorrectly calls recordLegacyDuration instead of recordStorageDuration, causing storage failures/latency to be recorded in the wrong histogram."
+        },
+        {
+          "golden_comment": "Inconsistency: using name instead of options.Kind for metrics recording differs from other methods.",
+          "severity": "Medium",
+          "matched_candidate": "The Delete success path passes the per-object name as the third metric argument even though that argument maps to the kind label, creating a separate histogram series for each deleted object and preventing kind-level aggregation; it should pass options.Kind.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same issue: metrics recording in the Delete success path uses the object name where the kind label should use options.Kind, causing inconsistent/incorrect metric labeling compared to other methods."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The asynchronous legacy writes in Create, Delete, Update, and DeleteCollection derive their 10-second timeout context from the request context, so request completion or cancellation can cancel the fire-and-forget legacy operation before its timeout and leave legacy storage stale; the timeout should be based on a context detached from request cancellation."
+        },
+        {
+          "candidate": "The asynchronous d.Legacy.DeleteCollection call is measured with startLegacy but recorded via recordStorageDuration, which misroutes legacy latency and failures into the storage histogram and leaves the legacy DeleteCollection metric series incomplete; it should use recordLegacyDuration."
+        },
+        {
+          "candidate": "The Delete method lacks a leading comment describing that it deletes from Storage synchronously and schedules the LegacyStorage deletion asynchronously, making it inconsistent with the documentation pattern used by the other DualWriterMode3 methods."
+        },
+        {
+          "candidate": "The asynchronous legacy writes in Create, Delete, Update, and DeleteCollection capture returned errors but only record them as metric labels, discarding them without logging; legacy divergence can therefore occur without a corresponding production log entry."
+        },
+        {
+          "candidate": "Create, Delete, Update, and DeleteCollection launch asynchronous legacy-write goroutines without any shared concurrency limit, so bursts of successful write requests can create many concurrent legacy calls and retain request objects and callbacks until the calls finish or time out; concurrency should be bounded with a semaphore or worker queue."
+        },
+        {
+          "candidate": "The d.Storage.Create error path uses startStorage but records the failure through recordLegacyDuration, creating a legacy metric observation even though no legacy operation ran; it should use recordStorageDuration."
+        },
+        {
+          "candidate": "The Create, Delete, Update, and DeleteCollection tests do not synchronize with the asynchronous legacy operation or assert mock expectations before returning, so a legacy goroutine that never runs can still leave the tests passing and the mode-3 legacy-write behavior unverified."
+        }
+      ],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 10,
+      "total_scored_candidates": 10,
+      "total_golden": 3,
+      "total_ignored_golden": 0,
+      "tp": 3,
+      "fp": 7,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.3,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "grafana__grafana__deputydev-v5__PR90045__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR90045__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/grafana/grafana/pull/106778": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "RuleActionsButtons is invoked with only promRule, but SilenceGrafanaRuleDrawer inside RuleActionsButtons still depends on a Grafana Ruler rule being present, so for Grafana rules coming from list views the 'Silence notifications' menu entry (now driven by Grafana Prom abilities) will toggle showSilenceDrawer without ever rendering the drawer. This means clicking 'Silence notifications' for these rules has no visible effect, even when abilities indicate silencing is allowed.",
+          "severity": "High",
+          "matched_candidate": "In the new list path, `RuleActionsButtons` receives only `promRule`, so `rulerRuleType.grafana.alertingRule(rule)` is false. Although the menu can grant the Silence action through the Prometheus ability and set `showSilenceDrawer`, the drawer is guarded by the ruler-only condition and also requires a ruler DTO, so the visible Silence action becomes clickable but opens no UI for prom-only rules.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same issue: in the list path RuleActionsButtons only has promRule, the Silence action can be shown/enabled via Prometheus/Grafana Prom abilities, but the SilenceGrafanaRuleDrawer rendering is still gated on having a Grafana Ruler rule/DTO, so clicking the action toggles state without displaying any drawer UI."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "In AlertRuleMenu.tsx, the Prometheus ability path can set canPause or canDelete to true when only promRule is available, but the menu rendering guards still require rulerRule: Pause checks rulerRuleType.grafana.rule(rulerRule), and Delete checks canDelete && rulerRule. As a result, Pause and Delete are treated as available by permissions but are not exposed for prom-only rules in the new list path."
+        },
+        {
+          "candidate": "In useAbilities.ts, the `// duplicate` comment above `const { isEditable, isRemovable, loading } = useIsGrafanaPromRuleEditable(rule);` is unclear because it does not identify what is duplicated or explain any intended follow-up, obscuring the purpose of the permission check for future maintainers."
+        },
+        {
+          "candidate": "In useAbilities.ts, `useAllGrafanaPromRuleAbilities` subscribes to the same folder twice for each rule row: `useIsGrafanaPromRuleEditable(rule)` calls `useFolder(folderUid)`, and `useCanSilenceInFolder(folderUid)` reaches another `useFolder(folderUid)` through `useFolderPermissions`. This creates redundant per-row folder subscriptions and unnecessary work in the rule list."
+        },
+        {
+          "candidate": "In RuleActionsButtons.V2.tsx, the warning emitted when identifier construction fails is identical for every failure and does not include context from `groupIdentifier`, `rule`, or `promRule`, so production logs cannot identify which rule or group caused the conversion failure."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The rendered GrafanaRuleListItem is missing the required key prop for React list items. This can cause rendering issues when the list order changes.",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 5,
+      "total_scored_candidates": 5,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 4,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.2,
+      "recall": 0.5,
+      "tool": "deputydev-v1",
+      "repo_name": "grafana__grafana__deputydev-v5__PR106778__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR106778__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/grafana/grafana/pull/107534": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "The applyTemplateVariables method is called with request.filters as the third parameter, but this parameter is not used in the corresponding test setup.",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 0,
+      "total_golden": 1,
+      "total_ignored_golden": 0,
+      "tp": 0,
+      "fp": 0,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "deputydev-v1",
+      "repo_name": "grafana__grafana__deputydev-v5__PR107534__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR107534__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/grafana/grafana/pull/79265": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Returning ErrDeviceLimitReached when no rows were updated is misleading; the device might not exist.",
+          "severity": "Low",
+          "matched_candidate": "`RowsAffected() == 0` after the UPDATE is treated as proof that a new device is blocked by the device limit, but the UPDATE also filters on `updated_at BETWEEN ...`; an existing device with a stored timestamp outside that window affects zero rows and is incorrectly denied with `ErrDeviceLimitReached`, causing authentication denial. The code should distinguish an unknown device from a timestamp mismatch, or update any row matching the existing `device_id`, and only return the limit error when the device is genuinely absent while the configured limit is full.",
+          "confidence": 0.84,
+          "reasoning": "The candidate flags the same core problem: interpreting RowsAffected()==0 as a device-limit condition and returning ErrDeviceLimitReached is misleading because zero updated rows can have other causes. Although it emphasizes a timestamp-window mismatch rather than simply a nonexistent device, it still identifies the same erroneous error handling around zero-row UPDATE results."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "`anonymousDeviceExpiration` is independently defined as 30 days in both the API package and `anonstore`, even though the values govern the same expiration policy for device listing and device-limit enforcement; changing only one constant would silently make the listing window and limit-enforcement window inconsistent. The expiration should be defined once and reused in both packages."
+        },
+        {
+          "candidate": "`tagDeviceUI` writes the device marker to the local cache before calling `CreateOrUpdateDevice`; if `CreateOrUpdateDevice` returns `ErrDeviceLimitReached`, the request is blocked but the cache entry remains, so the next request for the same device succeeds from the cache without checking the store and bypasses device-limit enforcement. The device should only be cached after `CreateOrUpdateDevice` succeeds."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Race condition: Multiple concurrent requests could pass the device count check simultaneously and create devices beyond the limit. Consider using a database transaction or lock.",
+          "severity": "High"
+        },
+        {
+          "golden_comment": "Anonymous authentication now fails entirely if anonDeviceService.TagDevice returns ErrDeviceLimitReached. Previously, device tagging was asynchronous and non-blocking. This change prevents anonymous users from authenticating when the device limit is reached.",
+          "severity": "Medium"
+        },
+        {
+          "golden_comment": "This call won’t compile: dbSession.Exec(args...) is given a []interface{} where the first element is the query, but Exec’s signature requires a first parameter of type string (not an interface{} splat).",
+          "severity": "Medium"
+        },
+        {
+          "golden_comment": "Time window calculation inconsistency: Using device.UpdatedAt.UTC().Add(-anonymousDeviceExpiration) as the lower bound but device.UpdatedAt as the current time may not match the intended logic. Consider using time.Now().UTC() consistently.",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 3,
+      "total_scored_candidates": 3,
+      "total_golden": 5,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 2,
+      "fn": 4,
+      "errors_count": 0,
+      "precision": 0.3333333333333333,
+      "recall": 0.2,
+      "tool": "deputydev-v1",
+      "repo_name": "grafana__grafana__deputydev-v5__PR79265__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR79265__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "`FallbackLocaleList` subclasses `Hash` while overriding `[]` to always compute a fallback locale list instead of reading Hash storage, so values stored through inherited Hash operations cannot be retrieved through lookup; this breaks Hash semantics and makes inherited storage operations inconsistent with `[]`, so it should be a plain class exposing only the required `[]` and `ensure_loaded!` methods."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Thread-safety issue with lazy @loaded_locales",
+          "severity": "Low"
+        },
+        {
+          "golden_comment": "Consider normalizing the input locale (e.g., to a symbol) when checking/loading here to avoid double-loading if the same locale is passed as a String vs Symbol (also applies to other locations in the PR).",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 1,
+      "total_scored_candidates": 1,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 0,
+      "fp": 1,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "deputydev-v1",
+      "repo_name": "discourse__discourse-graphite__deputydev-v5__PR9__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR9__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/grafana/grafana/pull/76186": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "`TestLogger.FromContext` returns a fresh `TestLogger`, so calls such as `m.logger.FromContext(ctx).Info(...)` write to a throwaway logger instead of the injected test logger, preventing assertions on the original logger's `InfoLogs`, `WarnLogs`, and other stores from observing context-derived log calls."
+        },
+        {
+          "candidate": "The new contextual logger middleware has no unit tests covering the endpoint attributes added by each instrumented method, optional user and datasource fields, or delegation behavior when optional `PluginContext` fields are nil."
+        },
+        {
+          "candidate": "`logRequest` relies on `ctx` having already been enriched by `ContextualLoggerMiddleware`, but this implicit middleware-order dependency is undocumented; if the ordering is wrong, emitted log entries lack plugin, endpoint, datasource, and user attributes that were previously added directly in `logRequest`."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) panic when a nil request is received. This occurs because they directly access req.PluginContext (via the instrumentContext function) without first checking if req is nil. This is a regression, as previous middleware layers gracefully handled nil requests.",
+          "severity": "High"
+        },
+        {
+          "golden_comment": "The traceID is no longer logged for plugin requests. During a refactoring, the tracing import and the logic to extract and add traceID from the context to log parameters were removed from the LoggerMiddleware. The newly introduced ContextualLoggerMiddleware does not add this information, resulting in missing traceID in plugin request logs and impacting debugging and request tracing capabilities.",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 3,
+      "total_scored_candidates": 3,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 0,
+      "fp": 3,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "deputydev-v1",
+      "repo_name": "grafana__grafana__deputydev-v5__PR76186__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/grafana__grafana__deputydev-v5__PR76186__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "NoMethodError before_validation in EmbeddableHost",
+          "severity": "Critical",
+          "matched_candidate": "The `before_validation` host-normalization callback calls `sub!` on `self.host` without checking that it is present, so saving an `EmbeddableHost` without a host raises `NoMethodError` before format validation can return a normal validation error.",
+          "confidence": 0.98,
+          "reasoning": "The candidate describes the same issue: a before_validation callback in EmbeddableHost calls a string method on a nil host, causing a NoMethodError instead of normal validation behavior."
+        },
+        {
+          "golden_comment": "The update and destroy methods in Admin::EmbeddableHostsController do not validate the existence of the EmbeddableHost record retrieved by ID. If EmbeddableHost.where(id: params[:id]).first returns nil (i.e., the host does not exist), attempting to call methods on the nil object (e.g., save_host or destroy) will result in a NoMethodError.",
+          "severity": "Medium",
+          "matched_candidate": "In `update`, an unknown or already-deleted `params[:id]` makes `EmbeddableHost.where(id: params[:id]).first` return `nil`, and passing that nil host into `save_host` dereferences it, causing a `NoMethodError`/500 instead of a not-found response.",
+          "confidence": 0.82,
+          "reasoning": "The candidate identifies the same nil-record dereference problem caused by `EmbeddableHost.where(id: params[:id]).first` returning nil, specifically in `update` via `save_host`. It omits the same issue in `destroy`, so it is incomplete but still matches the core underlying issue."
+        },
+        {
+          "golden_comment": "record_for_host compares lower(host) = ? but does not normalize the parameter’s case, so mixed‑case referer hosts may fail to match even though comparison intends to be case‑insensitive.",
+          "severity": "Medium",
+          "matched_candidate": "`record_for_host` lowercases the stored `host` value but not the supplied `uri.host` parameter, so under a case-sensitive comparison a mixed-case URL host fails to match an otherwise allowed host and valid hosts are rejected.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same issue: `record_for_host` applies `lower()` to the stored host but does not normalize the incoming `uri.host` parameter, causing mixed-case hosts to fail under case-sensitive comparison despite intended case-insensitive matching."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "In `destroy`, an unknown or already-deleted `params[:id]` makes `EmbeddableHost.where(id: params[:id]).first` return `nil`, and calling `nil.destroy` causes a `NoMethodError`/500 instead of a not-found response."
+        },
+        {
+          "candidate": "The migration interpolates host values from the free-text site setting directly into a raw SQL `INSERT`; a stored host containing a single quote can break the query and alter the SQL being executed, so each host value must be database-quoted or bound."
+        },
+        {
+          "candidate": "The migration accesses `result[0]['id']` without checking whether the category lookup returned a row; if the `embed_category` setting is absent or matches no category, `result[0]` is nil and the migration raises `NoMethodError` before reaching the uncategorized fallback."
+        },
+        {
+          "candidate": "`EmbeddableHost.record_for_host` repeatedly queries with `WHERE lower(host) = ?`, but the table has no matching functional index on `lower(host)`, so each host authorization lookup scans the table."
+        },
+        {
+          "candidate": "The controller spec only asserts inheritance and does not exercise the newly added CRUD behavior or error paths, leaving create, update, destroy, validation failures, authorization, and missing-record responses untested."
+        },
+        {
+          "candidate": "The fabricator definitions are in files named for the opposite model: `category_fabricator.rb` now defines only an embeddable host after removing the category definitions, making fabricators difficult to locate and breaking the existing file organization."
+        },
+        {
+          "candidate": "The deleted `embeddable_hosts` and `embed_category` setting keys still have translations in non-English server locale files such as `server.es.yml` and `server.fi.yml`, leaving orphaned locale entries that should be removed."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Because this migration inserts embeddable_hosts rows with raw SQL, any existing embeddable_hosts values that include http:// or /https:// or path segments won’t go through the EmbeddableHost model’s normalization, so the new host lookup (which compares only the bare host) may fail for migrated data. Consider ensuring that migrated hosts are normalized to the same format as newly created EmbeddableHost records so existing embedding configurations keep working.",
+          "severity": "High"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 10,
+      "total_scored_candidates": 10,
+      "total_golden": 4,
+      "total_ignored_golden": 0,
+      "tp": 3,
+      "fp": 7,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.3,
+      "recall": 0.75,
+      "tool": "deputydev-v1",
+      "repo_name": "discourse__discourse-graphite__deputydev-v5__PR10__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR10__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "In .topic-meta-data h5 a, the original code had color: scale-color($primary, $lightness: 30%) but was changed to dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%)). The lightness for the light theme changed from 30% to 70%, which is a dramatic inversion",
+          "severity": "Low",
+          "matched_candidate": "The `.topic-meta-data h5 a` color migration changes the light-theme branch from the original `scale-color($primary, $lightness: 30%)` to `70%`; because `dark-light-choose` uses the first branch on light themes, this makes the link substantially lighter than before and reduces its previous contrast instead of limiting the change to dark-theme rendering.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same change in `.topic-meta-data h5 a`: the light-theme color branch changed from `scale-color($primary, $lightness: 30%)` to `70%`, causing a much lighter color and contrast regression. This matches the golden issue."
+        },
+        {
+          "golden_comment": "This change for desktop/user.css changes $primary from 30% to 50% for the light theme; most other changes preserve the original $primary value and move the complement to $secondary for dark. Consider reviewing this (also applies to a similar .name change in the mobile variant).",
+          "severity": "Low",
+          "matched_candidate": "The `.group-member-info .name` color replacement in desktop and mobile user styles changes the light-theme value from the original `30%` to `50%`, making member names the same lightness as the adjacent `.title` and removing the previous visual hierarchy between name and title.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same underlying issue: the `.name` color change in desktop and mobile user styles alters the light-theme `$primary` value from 30% to 50%, rather than preserving the original primary value as expected. It also correctly notes the affected desktop/mobile `.name` context."
+        },
+        {
+          "golden_comment": "In topic-post.css the original code used $lightness: 70% but the replacement uses $lightness: 30% for the light theme. This makes the text significantly darker than intended.",
+          "severity": "Low",
+          "matched_candidate": "The mobile `.custom-message-length` color conversion replaces the original light-theme first branch `70%` with `30%`; because that branch controls light themes, the text becomes much darker and more prominent than before and diverges from the equivalent desktop rule that preserves `70%`.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same light-theme lightness regression: 70% was changed to 30%, making the text much darker than intended. Extra context about mobile/desktop does not change the underlying issue."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The documentation comment in `app/assets/stylesheets/common/base/discourse.scss` still describes the old table-cell coldmap default as `scale-color($primary, $lightness: 50%)` instead of the new theme-aware `dark-light-choose(...)` value, so it documents the wrong default expression."
+        },
+        {
+          "candidate": "The mobile topic-post `h3` color migration changes the light-theme first branch from the original `20%` lightness to `50%`, making the heading substantially lighter on light themes, removing its previous emphasis, and diverging from the equivalent desktop rule that keeps `20%`."
+        }
+      ],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 5,
+      "total_scored_candidates": 5,
+      "total_golden": 3,
+      "total_ignored_golden": 0,
+      "tp": 3,
+      "fp": 2,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.6,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "discourse__discourse-graphite__deputydev-v5__PR7__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR7__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "In the next action, capping the next offset at user_count can produce an empty page (e.g., total equal to limit results in offset == total, showing 2/2 with no members). This can cause confusing UX on the last page.",
+          "severity": "Medium",
+          "matched_candidate": "`Math.floor(user_count / limit) + 1` overcounts total pages when the member count is an exact multiple of the page size, creating a phantom empty page such as 50 members with a limit of 50 producing two pages; use `Math.ceil(user_count / limit)` instead.",
+          "confidence": 0.88,
+          "reasoning": "The candidate describes the same phantom empty-page bug for exact multiples of the page size, where pagination allows/displays an extra page with no members. Although it focuses on the total-page calculation rather than the next-offset cap, it identifies the same underlying UX issue."
+        },
+        {
+          "golden_comment": "HTTP method mismatch in .remove_member - test uses PUT but remove_member action expects DELETE",
+          "severity": "Medium",
+          "matched_candidate": "The controller spec uses PUT for `remove_member` even though the production route exposes the action through DELETE, so the test can pass by invoking the named action directly without exercising the route's DELETE method contract.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same issue: the spec/test uses PUT for remove_member while the actual route/action contract expects DELETE, allowing the test to bypass the correct HTTP method behavior."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "`removeMember` is called without handling its returned promise, so server-side deletion failures are discarded and the confirmation closes without notifying the administrator that the member was not removed."
+        },
+        {
+          "candidate": "`addMembers` is called without a rejection handler, so network or server-side failures receive no user-facing handling and leave the entered usernames in place without explaining that no members were added."
+        },
+        {
+          "candidate": "The loop calls `User.find_by_username` once for every submitted username, causing N separate user SELECT queries instead of resolving the submitted usernames with one bulk query before adding the users to the group."
+        },
+        {
+          "candidate": "The caller-provided `limit` is passed directly to the query without a maximum, so a request such as `limit=100000` can fetch and serialize a very large group in one response, imposing excessive database, memory, and serialization work."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": " The findMembers() call is now asynchronous and unhandled. The controller may not have member data immediately available, creating a race condition.",
+          "severity": "High"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 6,
+      "total_scored_candidates": 6,
+      "total_golden": 3,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 4,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.3333333333333333,
+      "recall": 0.6666666666666666,
+      "tool": "deputydev-v1",
+      "repo_name": "discourse__discourse-graphite__deputydev-v5__PR8__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR8__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.",
+          "severity": "Medium",
+          "matched_candidate": "`BlockedEmail.should_block?` updates `match_count` and `last_match_at` when called from `validate_each`, so repeated validations during one account creation attempt can mutate statistics multiple times and overcount actual registration attempts.",
+          "confidence": 0.92,
+          "reasoning": "The candidate identifies the same core issue: a method used to check whether an email should be blocked mutates statistics (`match_count`, `last_match_at`) as a side effect during validation/read-like usage. It focuses on overcounting rather than race conditions, but the underlying side-effect bug is the same."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "`rejectedEmails` is initialized as a mutable array on the controller prototype, so controller instances share rejected-address state; an email pushed by one instance can leak to another and cause that other instance to reject it."
+        },
+        {
+          "candidate": "The blocklist lookup compares the raw email string directly, so case-sensitive comparison can let differently cased versions of a blocked address, such as `BAD@spamclub.com` versus stored `bad@spamclub.com`, bypass the block."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 4,
+      "total_scored_candidates": 3,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 2,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.3333333333333333,
+      "recall": 0.5,
+      "tool": "deputydev-v1",
+      "repo_name": "discourse__discourse-graphite__deputydev-v5__PR3__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR3__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "-ms-align-items never existed in any version of IE/Edge; the correct legacy property is -ms-flex-align.",
+          "severity": "Low",
+          "matched_candidate": "`-ms-align-items` in the `align-items` mixin is not a valid CSS property and is ignored; since the valid prefixed declaration `-ms-flex-align` is already present, every use of the mixin emits a dead invalid vendor-prefixed property that should be removed.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same issue: `-ms-align-items` is not a valid IE/Edge-prefixed CSS property, and the correct legacy property is `-ms-flex-align`, so the invalid declaration should be removed."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "Converting `.small-action` to a flex container makes the existing `float: left` on the direct `.topic-avatar` child and `float: right` on the direct `button` child ineffective, so these declarations no longer control avatar or button placement; remove the dead floats and use flex layout such as `margin-left: auto` if the button must remain flush right."
+        },
+        {
+          "candidate": "The `//Flexbox` section header in `app/assets/stylesheets/common/foundation/mixins.scss` is missing the space used by the file's other section comments, such as `// Text`, and should be changed to `// Flexbox` for consistency."
+        },
+        {
+          "candidate": "The `align-items` mixin is formatted inconsistently with adjacent mixins by using four-space indentation and omitting the space after the colon in `align-items:$alignment`; format the block consistently with the rest of the file."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Mixing float: left with flexbox causes layout issues. Further this PR removes the float-based right alignment for .d-header .panel, which may cause the login panel in the non-Ember/noscript header (where .panel is nested inside .row and not a flex item) to stack under the title instead of remaining right-aligned.",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 4,
+      "total_scored_candidates": 4,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 3,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.25,
+      "recall": 0.5,
+      "tool": "deputydev-v1",
+      "repo_name": "discourse__discourse-graphite__deputydev-v5__PR5__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR5__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "Each user serialization redundantly parses the same website URL once before branching and again in branches that append the path, causing unnecessary URI parsing and allocations from unchanged input."
+        },
+        {
+          "candidate": "The sibling-subdomain logic repeatedly calls split on the same website and Discourse host values while comparing component lengths and suffixes, causing unnecessary allocation of equivalent host-component arrays on each serialization."
+        },
+        {
+          "candidate": "The equal-depth sibling-subdomain branch is untested: current serializer specs cover unrelated domains, exact matches, and parent domains, but do not exercise the elsif path where suffix comparison matches or does not match, leaving both return paths unprotected."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The include_website_name method is missing the required ? suffix. Rails serializers expect include_ methods to end with ? for conditional attribute inclusion, a convention followed by other methods in this serializer. Without it, the website_name attribute may not be conditionally included as intended. Additionally, the '.' << website_host string concatenation should be replaced with '.' + website_host or '.#{website_host}' to avoid mutating string literals, which can lead to issues.",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 3,
+      "total_scored_candidates": 3,
+      "total_golden": 1,
+      "total_ignored_golden": 0,
+      "tp": 0,
+      "fp": 3,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "deputydev-v1",
+      "repo_name": "discourse__discourse-graphite__deputydev-v5__PR6__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR6__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "SSRF vulnerability using open(url) without validation",
+          "severity": "Critical",
+          "matched_candidate": "`url` is fetched with unrestricted `open` after only a host check, without validating resolved addresses or redirect destinations against private, loopback, and link-local ranges or restricting scheme and redirects, so the server-side fetch can reach internal endpoints.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same SSRF risk from using `open(url)`/server-side fetching without sufficient URL validation, including internal/private targets and redirects."
+        },
+        {
+          "golden_comment": "The current origin validation using indexOf is insufficient and can be bypassed. An attacker could use a malicious domain like evil-discourseUrl.com to pass this check.",
+          "severity": "Medium",
+          "matched_candidate": "The message sender is validated with `discourseUrl.indexOf(e.origin)` instead of strict equality against the exact origin derived from `discourseUrl`, so a non-equal origin whose serialized value appears as a substring of `discourseUrl` can pass the check and set the iframe height.",
+          "confidence": 0.93,
+          "reasoning": "The candidate points to the same underlying bug: origin validation relies on substring matching via indexOf rather than strict exact-origin comparison, allowing crafted non-equal origins to bypass the check."
+        },
+        {
+          "golden_comment": "postMessage targetOrigin should be the origin (scheme+host+port), not the full referrer URL; using the full URL will cause the message to be dropped and prevent resizing.",
+          "severity": "Medium",
+          "matched_candidate": "The full `request.referer` is interpolated directly into a JavaScript string for the `postMessage` target origin; a full path is not a valid target origin, and an unescaped quote in the referer can break out of the string, which can prevent resize-message delivery or inject JavaScript into the iframe response.",
+          "confidence": 0.98,
+          "reasoning": "The candidate explicitly identifies that using the full request.referer, including the path, as the postMessage targetOrigin is invalid and can prevent resize-message delivery. This is the same underlying issue as the golden comment, though it also mentions an additional escaping/injection concern."
+        },
+        {
+          "golden_comment": "The ERB block closes with end if, which is invalid Ruby/ERB and will raise at render; it should just be end to close the if block.",
+          "severity": "Medium",
+          "matched_candidate": "The template uses `<%- end if %>` with a trailing `if` that has no modifier condition, generating invalid Ruby and causing the embed template to raise a syntax error instead of rendering the embedded topic view.",
+          "confidence": 0.99,
+          "reasoning": "The candidate identifies the same invalid ERB/Ruby syntax: using `<%- end if %>` with a trailing `if` instead of simply closing the block with `end`, causing a render-time syntax error."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "`i.content` is used without being guarded before calling `scrub`; feed items that omit `content` raise `NoMethodError`, abort processing of the remaining items, and the polling job has retries disabled."
+        },
+        {
+          "candidate": "Feed item HTML is passed to `TopicEmbed.import` after only `CGI.unescapeHTML`, which decodes entities but does not sanitize; because the imported value is later stored as `raw_html` with validations skipped and rendered with `raw`, attacker-controlled scripts or event-handler attributes from the feed can persist and render."
+        },
+        {
+          "candidate": "`url` is interpolated unescaped into the imported-from link's `href` attribute and link text; the HTTP-prefix check does not exclude quote characters, so a crafted URL can break out of the single-quoted `href` and inject attributes or markup that is stored as raw HTML and later rendered with `raw`."
+        },
+        {
+          "candidate": "`open(url).read` performs an unbounded remote read without explicit open/read timeouts or a maximum response size, buffering the full response before Readability processes it, so a slow or very large response can occupy the worker and consume excessive memory."
+        },
+        {
+          "candidate": "The throttle key is set with `setnx` and then expired in a separate command; if the process stops after `setnx` succeeds but before `expire`, the key remains without an expiration and that URL is throttled permanently."
+        },
+        {
+          "candidate": "`TopicRetriever` directly executes the scheduled `PollFeed` job on an unknown embed URL when feed polling is enabled, synchronously downloading and parsing the entire feed and importing every item, which blocks the retrieval worker with network and database work."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The code sets X-Frame-Options: ALLOWALL which completely disables clickjacking protection. The referer validation can be bypassed (referer headers are easily spoofed), and the fallback to empty string for nil referer masks validation failures.",
+          "severity": "Medium"
+        },
+        {
+          "golden_comment": "The TopicEmbed.import method is susceptible to a NoMethodError if the contents parameter is nil when attempting to append a string, and an XSS vulnerability due to unescaped url interpolation in the generated HTML.",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 10,
+      "total_scored_candidates": 10,
+      "total_golden": 6,
+      "total_ignored_golden": 0,
+      "tp": 4,
+      "fp": 6,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.4,
+      "recall": 0.6666666666666666,
+      "tool": "deputydev-v1",
+      "repo_name": "discourse__discourse-graphite__deputydev-v5__PR4__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR4__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The downsize method is defined twice. The second definition, which expects a single dimensions string parameter, overrides the first, which expected separate max_width and max_height parameters. This makes the original method unreachable and breaks existing code that calls it with separate width and height arguments.",
+          "severity": "Medium",
+          "matched_candidate": "Defining self.downsize a second time in Ruby replaces the existing five-argument downsize(from, to, max_width, max_height, opts={}) method, so the existing resize_emoji.rb caller that still invokes OptimizedImage.downsize(path, path, 100, 100, opts) will raise ArgumentError at runtime because the surviving method accepts at most four arguments.",
+          "confidence": 0.99,
+          "reasoning": "The candidate identifies the same underlying issue: a second Ruby definition of `downsize` overrides the original method that accepted separate width and height arguments, causing existing callers using the old signature to fail at runtime."
+        },
+        {
+          "golden_comment": "Hardcoding maxSizeKB = 10 * 1024 ignores Discourse.SiteSettings['max_' + type + '_size_kb'], so the client-side limit can diverge from server-side and per-type settings (also applies to the 413 handler below).",
+          "severity": "Low",
+          "matched_candidate": "Replacing the previous per-type upload size checks with a uniform 10 MB limit means configured limits such as max_<type>_size_kb no longer drive client validation, so attachment and avatar uploads with lower configured limits can be allowed through and the existing test that sets max_image_size_kb to 5 KB no longer rejects a 10 KB file as expected.",
+          "confidence": 0.97,
+          "reasoning": "The candidate identifies the same issue: a hardcoded 10 MB client-side upload limit replaces/ignores the per-type max_<type>_size_kb SiteSettings, causing divergence from configured/server-side limits. It does not explicitly mention the 413 handler, but the core underlying bug matches."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The upload controller ignores the return value of OptimizedImage.downsize and does not verify the final file size before calling Upload.create_for, so failed or insufficient ImageMagick conversions can be treated as successful, repeated unnecessarily, and an image that remains above max_image_size_kb can continue into upload creation."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Passing 80% as the dimensions can fail for animated GIFs when allow_animated_thumbnails is true, since the animated path uses gifsicle --resize-fit which expects WxH geometry, not a percentage; downsizing would then silently fail.",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 3,
+      "total_scored_candidates": 3,
+      "total_golden": 3,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 1,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.6666666666666666,
+      "recall": 0.6666666666666666,
+      "tool": "deputydev-v1",
+      "repo_name": "discourse__discourse-graphite__deputydev-v5__PR1__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR1__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "logic: Potential nil pointer exception - if no TopicUser record exists, tu will be nil and calling methods on it will crash",
+          "severity": "High",
+          "matched_candidate": "`TopicUser.find_by(user_id: current_user.id, topic_id: params[:topic_id])` can return `nil`, but the result is dereferenced with `tu.notification_level` unconditionally, causing a `NoMethodError` and a 500 response when the user/topic pair has no `TopicUser` row. Handle the missing `TopicUser` before reading `tu.notification_level`.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same underlying issue: `find_by` may return nil when no TopicUser record exists, and dereferencing `tu.notification_level` would cause a nil pointer/NoMethodError crash."
+        },
+        {
+          "golden_comment": "Typo in property name: 'stopNotificiationsText' should be 'stopNotificationsText' (missing 'n' in 'Notifications')",
+          "severity": "Low",
+          "matched_candidate": "The controller/template binding uses the misspelled identifier `stopNotificiationsText`; because the controller defines it and the Handlebars template references it verbatim, this typo becomes the property name consumers must use and will require a coordinated rename later. Rename it to `stopNotificationsText` in both places.",
+          "confidence": 1.0,
+          "reasoning": "The candidate explicitly identifies the same typo in the property name `stopNotificiationsText` and recommends renaming it to `stopNotificationsText`, matching the golden comment exactly in substance."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The `%{respond_instructions}` footer is rendered inside the `context_posts.present?` block and also rendered unconditionally later in the template, so emails with context posts receive duplicate response instructions. Remove the placeholder from the `context_posts` block."
+        },
+        {
+          "candidate": "The unsubscribe endpoint performs a persistent notification-state mutation via GET even though the URL is included in notification emails; automated link followers such as mail scanners or preview tools can therefore change a user's notification level without user intent. Use GET only for a confirmation page and perform the mutation via POST or PATCH."
+        }
+      ],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 4,
+      "total_scored_candidates": 4,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 2,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "discourse__discourse-graphite__deputydev-v5__PR2__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/discourse__discourse-graphite__deputydev-v5__PR2__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/calcom/cal.com/pull/22532": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The updateManyByCredentialId call uses an empty data object, which prevents Prisma's @updatedAt decorator from updating the updatedAt timestamp. This results in inaccurate cache status tracking, as the timestamp isn't updated when the cache is refreshed. To fix this, explicitly set the updatedAt field.",
+          "severity": "Medium",
+          "matched_candidate": "`updateManyByCredentialId` forwards an empty update object directly to Prisma's `updateMany`, so no field is written and `@updatedAt` is not bumped, leaving `SelectedCalendar.updatedAt` stale after availability is refreshed instead of recording the cache-refresh timestamp.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same issue: an empty update object passed through updateManyByCredentialId to Prisma updateMany prevents @updatedAt from being bumped, leaving the cache/SelectedCalendar timestamp stale after refresh."
+        },
+        {
+          "golden_comment": "logic: macOS-specific sed syntax with empty string after -i flag will fail on Linux systems",
+          "severity": "Low",
+          "matched_candidate": "The script uses BSD/macOS-specific `sed -i ''` syntax when updating an existing `GOOGLE_WEBHOOK_URL`, which GNU sed on Linux interprets incorrectly and can terminate the script instead of updating the environment file.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same portability bug: using macOS/BSD-specific `sed -i ''` syntax, which fails or behaves incorrectly with GNU sed on Linux."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "After deleting the cache, `viewer.calendars.connectedCalendars` is not invalidated, so the cached tRPC query result can continue showing the deleted `cacheUpdatedAt` value and stale cache status, unlike the disconnect mutation which explicitly invalidates the query."
+        },
+        {
+          "candidate": "The missing or inaccessible credential path throws a plain `Error`, which tRPC reports as `INTERNAL_SERVER_ERROR`, so an expected ownership lookup failure is exposed to the client with incorrect HTTP/tRPC semantics instead of an explicit `NOT_FOUND` or `UNAUTHORIZED` `TRPCError`."
+        }
+      ],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 4,
+      "total_scored_candidates": 4,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 2,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR22532__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR22532__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/calcom/cal.com/pull/8330": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Incorrect end time calculation using slotStartTime instead of slotEndTime",
+          "severity": "Medium",
+          "matched_candidate": "In `packages/trpc/server/routers/viewer/slots.ts` line 142, the working-hours fallback computes `end` from `slotStartTime` instead of `slotEndTime`, making `start` and `end` identical. As a result, `end > workingHour.endTime` checks the slot start rather than the actual slot end, so slots that start within working hours but finish after the working-hours boundary are incorrectly allowed.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same bug: the end time is calculated from slotStartTime instead of slotEndTime, causing incorrect working-hours boundary validation for slots that extend past the end time."
+        },
+        {
+          "golden_comment": "Using === for dayjs object comparison will always return false as it compares object references, not values. Use .isSame() method instead: dayjs(date.start).add(utcOffset, 'minutes').isSame(dayjs(date.end).add(utcOffset, minutes))",
+          "severity": "Medium",
+          "matched_candidate": "In `packages/trpc/server/routers/viewer/slots.ts` line 114, the zero-length date override guard compares two separately constructed Dayjs objects with `===`: `dayjs(date.start).add(...) === dayjs(date.end).add(...)`. Because JavaScript object equality checks identity rather than timestamp value, this condition is always false for distinct Dayjs instances, making the zero-duration override branch unreachable and causing zero-length overrides to be silently ignored; use Dayjs value comparison such as `.isSame()` instead.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same issue: comparing separately created Dayjs objects with `===` checks object identity rather than date/time value, making the condition always false, and recommends using `.isSame()` instead."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "In `packages/trpc/server/routers/viewer/slots.ts`, the flattened `dateOverrides` array containing overrides for all hosts is passed into `checkIfIsAvailable`, but the date-override predicate does not filter overrides by the current user being checked. In multi-host scenarios such as collective or round-robin events, a date override belonging to host A can incorrectly suppress or activate availability for host B, producing incorrect slot availability results."
+        },
+        {
+          "candidate": "In `apps/web/test/lib/getSchedule.test.ts` line 798, the assertion comment `// it should return the same as this is the utc time` is misleading because the test is actually verifying that UTC slot times remain invariant across attendee timezones for date override boundaries. The vague comment obscures the timezone-offset invariant being tested and may cause future maintainers to misread the test as a no-op UTC assertion."
+        },
+        {
+          "candidate": "In `packages/lib/slots.ts` line 218, `getSlots` repeatedly constructs identical Dayjs values inside the `activeOverrides.flatMap` callback: `dayjs(override.start).utc().add(offset, \"minute\")` is computed twice for `startTime`, and `dayjs(override.end).utc().add(offset, \"minute\")` is computed twice for `endTime`. This creates avoidable parsing/wrapping/allocation overhead for every override across scheduling loops; cache the adjusted start and end Dayjs values once per override."
+        },
+        {
+          "candidate": "In `packages/trpc/server/routers/viewer/slots.ts` line 106, the `dateOverrides.find()` callback in `checkIfIsAvailable` repeatedly constructs the same adjusted Dayjs values for each override during every slot availability check. `dayjs(date.start).add(utcOffset, \"minutes\")` is computed up to three times and `dayjs(date.end).add(utcOffset, \"minutes\")` up to two times per iteration, with a timezone lookup also repeated, creating avoidable O(slots × hosts × overrides) allocation and parsing cost; cache the adjusted start and end values inside the callback."
+        },
+        {
+          "candidate": "In `packages/trpc/server/routers/viewer/slots.ts` line 87, the `checkIfIsAvailable` `dateOverrides` parameter type only declares `{ start: Date; end: Date }[]` even though the flattened array passed at runtime includes per-user `userId` and `timeZone` fields. Because the function iterates the full flattened array without filtering by the current host/user, an override belonging to one user can incorrectly affect another user's availability in fixed-host or other multi-user scheduling paths."
+        },
+        {
+          "candidate": "In `packages/trpc/server/routers/viewer/slots.ts` line 87, the `checkIfIsAvailable` `dateOverrides` type omits the `timeZone` and `userId` fields that are added to the flattened date override entries in `getSchedule`. This makes the type out of sync with the actual data shape and prevents the predicate from using per-override timezone or user information for correct conversion and filtering."
+        }
+      ],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 13,
+      "total_scored_candidates": 8,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 6,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.25,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR8330__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR8330__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/calcom/cal.com/pull/14943": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can lose increments under concurrency; consider an atomic increment via Prisma (increment: 1) to avoid race conditions (also applies to the similar update in the catch block).",
+          "severity": "High",
+          "matched_candidate": "The retry count update uses `reminder.retryCount + 1` based on a value fetched before processing, including in the catch path, so overlapping cron executions can read the same stale retry count and overwrite each other with the same incremented value; this lost increment can delay retry-based cleanup because multiple failures are not all recorded.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same stale read/write race on retryCount using reminder.retryCount + 1, including the catch path, and explains lost increments under overlapping executions. This matches the golden issue, though it does not explicitly mention Prisma's atomic increment fix."
+        },
+        {
+          "golden_comment": "The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (e.g., Email, WhatsApp) that have retryCount > 1. This occurs because the retryCount condition within the OR clause for deletion lacks a method: WorkflowMethods.SMS filter, causing it to apply to all reminder types instead of only SMS reminders, which is the intended scope of this function.",
+          "severity": "High",
+          "matched_candidate": "In `scheduleSMSReminders.ts`, the retry-count branch of `prisma.workflowReminder.deleteMany` lacks `method: WorkflowMethods.SMS`, so it deletes any `WorkflowReminder` with `retryCount > 1` regardless of delivery method; this can incorrectly delete reminders belonging to non-SMS workflows even though the surrounding cleanup and scheduling logic is SMS-specific.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same bug: the retryCount > 1 deletion branch in scheduleSMSReminders.ts is missing the SMS method filter, causing non-SMS workflow reminders to be deleted despite the logic being SMS-specific."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 2,
+      "total_scored_candidates": 2,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 0,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR14943__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR14943__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/calcom/cal.com/pull/22345": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "In getBaseConditions(), the else if (filterConditions) and final else branches are unreachable. This is because getAuthorizationConditions() always returns a non-null Prisma.Sql object, making authConditions always truthy, which means only the first two if/else if conditions are ever evaluated.",
+          "severity": "Low",
+          "matched_candidate": "`getAuthorizationConditions()` returns a non-nullable `Prisma.Sql`, so `authConditions` is always truthy; the current `getBaseConditions()` control flow includes filter-only and final `NOTHING_CONDITION` branches that are unreachable under the method contract, misleadingly advertising states that cannot occur and should be simplified to only branch on `filterConditions`.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same issue: `getAuthorizationConditions()` always returns a non-null `Prisma.Sql`, making `authConditions` always truthy and rendering the filter-only and final `NOTHING_CONDITION` branches in `getBaseConditions()` unreachable."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "Caching regression coverage was removed even though `InsightsBookingService` still memoizes authorization and filter conditions via `cachedAuthConditions` and `cachedFilterConditions`; without equivalent tests using strict reference equality on consecutive calls to the new `Prisma.Sql` results, regressions that rebuild conditions instead of returning the cached object would no longer be detected."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Fetching userIdsFromOrg only when teamsFromOrg.length > 0 can exclude org-level members for orgs without child teams; consider deriving from teamIds (which includes orgId) or removing the guard so org-only orgs still include member user bookings.",
+          "severity": "Medium"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 2,
+      "total_scored_candidates": 2,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 1,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.5,
+      "tool": "deputydev-v1",
+      "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR22345__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR22345__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/calcom/cal.com/pull/11059": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The parseRefreshTokenResponse function incorrectly sets refresh_token to the hardcoded string 'refresh_token' when it's missing from the OAuth refresh token response. This invalidates the token, breaking subsequent token refreshes and causing authentication failures.",
+          "severity": "High",
+          "matched_candidate": "When the OAuth refresh response omits `refresh_token`, the parser substitutes the literal string `\"refresh_token\"`, allowing callers to persist that placeholder over a valid stored refresh token and causing later provider refreshes to fail.",
+          "confidence": 1.0,
+          "reasoning": "The candidate describes the same bug: when the OAuth refresh response lacks a refresh_token, the parser uses the literal string \"refresh_token\", which can overwrite/persist an invalid token and break future refresh/authentication flows."
+        },
+        {
+          "golden_comment": "Invalid Zod schema syntax. Computed property keys like [z.string().toString()] are not valid in Zod object schemas and will cause runtime errors. ",
+          "severity": "High",
+          "matched_candidate": "The Zod schema uses computed keys with `toString()`, which creates fixed property names rather than wildcard fields, so it does not validate arbitrary numeric expiry fields or preserve arbitrary token fields as intended in sync mode.",
+          "confidence": 0.88,
+          "reasoning": "The candidate points to the same misuse of computed Zod schema keys with `toString()`. Although it emphasizes incorrect fixed property names and failed wildcard validation rather than runtime errors, it identifies the same underlying schema bug."
+        },
+        {
+          "golden_comment": "parseRefreshTokenResponse returns a Zod safeParse result ({ success, data, error }), not the credential key object. Persisting that as key stores the wrapper instead of the token payload; we should store the parsed data or use schema parse.",
+          "severity": "High",
+          "matched_candidate": "Google Calendar assigns the entire `{ success: true, data: ... }` result from `parseRefreshTokenResponse` directly to the credential `key`, causing Prisma to persist the validation wrapper instead of the Google credential object.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same underlying bug: the safeParse wrapper returned by parseRefreshTokenResponse is being assigned/persisted as the credential key instead of the parsed token data."
+        },
+        {
+          "golden_comment": "When APP_CREDENTIAL_SHARING_ENABLED and CALCOM_CREDENTIAL_SYNC_ENDPOINT are set, the refreshFunction helper returns the fetch Response, but several callers (for example GoogleCalendarService.refreshAccessToken expecting res.data, and HubspotCalendarService.refreshAccessToken expecting a HubspotToken) assume it returns the integration-specific token object. That mismatch will cause runtime errors in the sync-enabled path unless the return type or those call sites are adjusted.",
+          "severity": "High",
+          "matched_candidate": "In sync mode, `refreshOAuthTokens` returns a raw Fetch `Response` even though callers expect provider-specific token objects, causing sync-mode refreshes to produce undefined fields or runtime errors when callers read shapes such as `.data` or `HubspotToken` fields.",
+          "confidence": 0.97,
+          "reasoning": "The candidate identifies the same sync-enabled path bug: a helper returns a raw fetch Response while callers expect provider-specific token payloads such as `.data` or Hubspot token fields, leading to runtime errors."
+        },
+        {
+          "golden_comment": "When the sync endpoint path is used, res is a fetch Response and has no .data; res?.data will be undefined and token.access_token will throw at runtime. This relies on a consistent return shape from refreshOAuthTokens, which isn’t guaranteed currently.",
+          "severity": "High",
+          "matched_candidate": "In sync mode, `refreshOAuthTokens` returns a raw Fetch `Response` even though callers expect provider-specific token objects, causing sync-mode refreshes to produce undefined fields or runtime errors when callers read shapes such as `.data` or `HubspotToken` fields.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same underlying bug: in the sync path, refreshOAuthTokens returns a raw fetch Response rather than the expected token data shape, so callers accessing fields like .data or access_token can get undefined/runtime errors."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The user and app existence checks in the app credential webhook are awaited sequentially even though neither query depends on the other, adding unnecessary serial database latency to every credential sync invocation."
+        },
+        {
+          "candidate": "The webhook handler calls `schema.parse(req.body)`, so malformed or missing request bodies throw an unhandled Zod exception and escape as a 500 instead of returning a controlled 400 validation response."
+        },
+        {
+          "candidate": "The decrypted credential payload is passed to `JSON.parse` without error handling, so malformed decrypted JSON can cause an uncontrolled exception."
+        },
+        {
+          "candidate": "The decrypted credential payload is persisted to `credential.key` after any successful JSON parse without validating that it matches the selected app's expected credential structure, allowing structurally invalid credentials to be stored."
+        },
+        {
+          "candidate": "The credential-sync POST identifies a Cal.com user and app but sends no authorization or shared-secret information, leaving the external endpoint without an application-level way to verify that the request originated from this service."
+        },
+        {
+          "candidate": "The OAuth refresh path awaits the external credential-sync `fetch` without an abort timeout, so an unresponsive sync endpoint can indefinitely stall calendar or video operations waiting for token refresh."
+        },
+        {
+          "candidate": "The credential-sync helper returns the Fetch response without checking `response.ok`, so HTTP 4xx or 5xx responses are treated as successful token refreshes and only fail later while parsing or using the response."
+        },
+        {
+          "candidate": "The Salesforce refresh code contains a failure branch for a `parseRefreshTokenResponse` result even though the helper throws on validation failure and only returns successful results, making that branch unreachable and inconsistent with the helper contract."
+        },
+        {
+          "candidate": "Zoho Bigin passes `credentialId` as the helper's third argument, which is serialized as `calcomUserId`, so sync-mode requests send a credential row ID in the field reserved for the Cal.com user ID and identify the wrong user."
+        }
+      ],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 13,
+      "total_scored_candidates": 14,
+      "total_golden": 5,
+      "total_ignored_golden": 0,
+      "tp": 5,
+      "fp": 9,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.35714285714285715,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR11059__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR11059__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/calcom/cal.com/pull/7232": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Asynchronous functions deleteScheduledEmailReminder and deleteScheduledSMSReminder are called without await inside forEach loops. This occurs during booking rescheduling/cancellation, and workflow/workflow step deletion/updates. Consequently, scheduled workflow reminders may not be reliably cancelled, potentially leaving them active.",
+          "severity": "Medium",
+          "matched_candidate": "In `handleCancelBooking.ts`, reminder cancellations are started inside nested `forEach` loops without awaiting the asynchronous `deleteScheduledEmailReminder` and `deleteScheduledSMSReminder` calls, so the booking cancellation handler can continue and respond before reminder rows have been cancelled or deleted.",
+          "confidence": 0.86,
+          "reasoning": "The candidate identifies the same core bug: asynchronous deleteScheduledEmailReminder/deleteScheduledSMSReminder calls are made inside forEach loops without awaiting them, so reminder cancellation may not complete reliably. It only mentions the booking cancellation instance rather than all contexts in the golden comment, but it matches the underlying issue."
+        },
+        {
+          "golden_comment": "When immediateDelete is true, the deleteScheduledEmailReminder function cancels the SendGrid email but fails to delete the corresponding WorkflowReminder record from the database. This creates orphaned database entries and is inconsistent with the immediateDelete: false path, which marks the record as cancelled. The SendGrid DELETE API call is also omitted in this path.",
+          "severity": "High",
+          "matched_candidate": "In `emailReminderManager.ts`, the `immediateDelete` branch cancels the SendGrid batch and then returns without deleting or updating the local `workflowReminder` row, leaving stale reminder records after an immediate deletion request.",
+          "confidence": 0.9,
+          "reasoning": "The candidate identifies the same core bug: the immediateDelete branch returns after handling SendGrid cancellation without deleting or updating the local WorkflowReminder row, leaving stale/orphaned reminder records. It does not explicitly mention the omitted SendGrid DELETE API call, but it matches the main underlying issue described in the golden comment."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "In `handleNewBooking.ts`, the reschedule flow uses `forEach` without awaiting the asynchronous reminder-cancellation helpers, so the surrounding `try/catch` can finish immediately and the reschedule can proceed while the previous booking's email or SMS reminders remain active."
+        },
+        {
+          "candidate": "In `scheduleEmailReminders.ts`, wrapping the entire reminder-cancellation loop in one `try/catch` means a single `client.request` failure aborts the rest of the batch, leaves later reminders active, and prevents already accumulated database deletions from being awaited."
+        },
+        {
+          "candidate": "In `WorkflowStepContainer.tsx`, the phone verification placeholder `\"Verification code\"` and button label `\"Verify\"` are hard-coded instead of using the `t(...)` i18n function, so those user-facing strings will always appear in English regardless of the user's locale."
+        }
+      ],
+      "false_negatives": [],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 5,
+      "total_scored_candidates": 5,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 3,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.4,
+      "recall": 1.0,
+      "tool": "deputydev-v1",
+      "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR7232__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR7232__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/calcom/cal.com/pull/14740": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Case sensitivity bypass in email blacklist",
+          "severity": "High",
+          "matched_candidate": "`guest` is not normalized before checking against lowercase blacklisted guest emails or existing attendees, so case variants can bypass the blacklist and case variants of existing attendee emails can be inserted as duplicate attendees.",
+          "confidence": 0.98,
+          "reasoning": "The candidate explicitly identifies that lack of normalization allows case variants of guest emails to bypass the lowercase email blacklist, which is the same underlying issue as the golden comment. The additional duplicate-attendee concern does not change the match."
+        },
+        {
+          "golden_comment": "The logic for checking team admin/owner permissions is incorrect. This condition uses AND (&&) which requires both isTeamAdmin AND isTeamOwner to be true, but it should use OR (||) since a user needs to be either an admin OR an owner to have permission.",
+          "severity": "Critical",
+          "matched_candidate": "The authorization check combines `isTeamAdmin` and `isTeamOwner` with `&&`, but `isTeamAdmin` already accepts ADMIN and OWNER while `isTeamOwner` accepts only OWNER, so ADMIN-only members who are neither organizer nor attendee are incorrectly denied permission to add guests.",
+          "confidence": 0.95,
+          "reasoning": "The candidate identifies the same core authorization bug: `isTeamAdmin` and `isTeamOwner` are combined with `&&`, causing users who are only admins to be denied, whereas admin or owner should be sufficient."
+        },
+        {
+          "golden_comment": "This calls the email sender with the original guests, so existing attendees included in the input will be treated as new when sending notifications, leading to incorrect emails.",
+          "severity": "Medium",
+          "matched_candidate": "`sendAddGuestsEmails` is called with the original `guests` input instead of `uniqueGuests`, so email-template routing can classify an already-existing submitted attendee as newly added and send the full booking-confirmation template instead of the guests-added notification.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same bug: the email sender is invoked with the original guests list rather than the filtered unique/new guests, causing existing attendees in the input to be treated as newly added and receive incorrect email notifications/templates."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "`ZAddGuestsInputSchema` is defined inside `AddGuestsDialog` even though it has no dependency on component state or props, causing the same static Zod schema object to be reconstructed on every render instead of being hoisted to module scope."
+        },
+        {
+          "candidate": "The organizer translation is awaited only after all attendee translations complete even though the operations are independent, adding avoidable serial translation latency and delaying calendar processing."
+        },
+        {
+          "candidate": "The email-delivery catch block logs only a context-free message and discards the caught error, leaving no stack, error details, booking ID, or user context to diagnose a swallowed `sendAddGuestsEmails` failure."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "uniqueGuests filters out existing attendees and blacklisted emails but does not deduplicate duplicates within the input; createMany can insert duplicate attendee rows if the client submits repeated emails.",
+          "severity": "Medium"
+        },
+        {
+          "golden_comment": "Starting with an array containing an empty string may cause validation issues. Consider starting with an empty array [] and handling the empty state in the MultiEmail component instead.",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 6,
+      "total_scored_candidates": 6,
+      "total_golden": 5,
+      "total_ignored_golden": 0,
+      "tp": 3,
+      "fp": 3,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.6,
+      "tool": "deputydev-v1",
+      "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR14740__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR14740__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/calcom/cal.com/pull/10600": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The exported function TwoFactor handles backup codes and is in BackupCode.tsx. Inconsistent naming.",
+          "severity": "Low",
+          "matched_candidate": "In `apps/web/components/auth/BackupCode.tsx`, the default-exported component is declared as `TwoFactor` even though the file and imports identify it as `BackupCode`, making the component purpose unclear and causing confusing IDE name resolution or stack traces.",
+          "confidence": 0.98,
+          "reasoning": "The candidate identifies the same inconsistency: `BackupCode.tsx` exports a component named `TwoFactor`, which is confusing because the file/function naming does not match its backup-code purpose."
+        },
+        {
+          "golden_comment": "Error message mentions 'backup code login' but this is a disable endpoint, not login",
+          "severity": "Low",
+          "matched_candidate": "In `apps/web/pages/api/auth/two-factor/totp/disable.ts`, the `console.error` message says it \"cannot proceed with backup code login\" even though the endpoint is disabling 2FA, which can mislead engineers triaging logs for this endpoint.",
+          "confidence": 1.0,
+          "reasoning": "The candidate identifies the same mismatch: the error log refers to 'backup code login' even though the file/endpoint is for disabling TOTP/2FA, causing misleading logs."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "In `apps/web/components/settings/EnableTwoFactorModal.tsx`, the Copy button calls `navigator.clipboard.writeText()` without awaiting or catching the returned Promise, so the success toast is shown even when clipboard copying fails due to an unavailable Clipboard API, non-HTTPS context, denied permission, or another rejection, misleading users into believing their backup codes were copied."
+        },
+        {
+          "candidate": "In `packages/features/auth/lib/next-auth-options.ts`, password verification is skipped whenever `credentials.totpCode` is non-empty because the password check is gated by `if (user.password && !credentials.totpCode)`. If an attacker submits both `totpCode` and `backupCode`, the presence of `totpCode` skips the password check while the backup-code branch can still grant access, allowing a crafted login request to bypass password verification."
+        },
+        {
+          "candidate": "In `apps/web/components/settings/EnableTwoFactorModal.tsx`, `URL.createObjectURL` is used to create a backup-code download URL after setup, but the URL is only revoked on the next setup attempt and not when the modal is closed, canceled, reset, or unmounted, so the Blob URL and backup-code text remain in memory for the lifetime of the page and can accumulate across modal openings."
+        },
+        {
+          "candidate": "The duplicated `apps/web/components/security/TwoFactorAuthAPI.ts` still exposes the old two-argument `disable(password, code)` signature while `apps/web/components/settings/TwoFactorAuthAPI.ts` was updated to accept a backup code, so the security-page disable flow cannot send `backupCode` to the API and users there cannot disable 2FA with a backup code."
+        },
+        {
+          "candidate": "The duplicated `apps/web/components/security/DisableTwoFactorModal.tsx` remains separate from the updated settings modal and still calls `TwoFactorAuthAPI.disable(password, totpCode)` without backup-code support, creating a maintenance hazard where security-critical 2FA changes must be applied to multiple divergent component copies."
+        },
+        {
+          "candidate": "In `apps/web/components/settings/EnableTwoFactorModal.tsx`, `backupCodes` is initialized with `useState([])` instead of an explicit `useState<string[]>([])`, which removes or weakens TypeScript safety for downstream uses such as `backupCodes.map(formatBackupCode)` and clipboard serialization, allowing API response shape or value mismatches to go uncaught by the compiler."
+        },
+        {
+          "candidate": "In `apps/web/components/settings/EnableTwoFactorModal.tsx`, `resetState()` resets the password, error message, and step but does not clear `backupCodes`, so after closing and reopening the modal while it remains mounted, stale backup codes from a previous setup session can remain in state and be displayed."
+        },
+        {
+          "candidate": "In `apps/web/components/settings/EnableTwoFactorModal.tsx`, `resetState()` does not revoke or clear `backupCodesUrl`, so the object URL created for backup-code download is not released when the modal is closed without a new setup attempt, leaking the Blob URL and its backup-code data."
+        },
+        {
+          "candidate": "In `apps/web/components/settings/EnableTwoFactorModal.tsx`, the Copy button ignores the Promise returned by `navigator.clipboard.writeText()` and always shows the success toast, so if clipboard access is denied, unavailable, blocked in an iframe, or otherwise rejects, users are falsely told their backup codes were copied."
+        },
+        {
+          "candidate": "The duplicated `apps/web/components/security/DisableTwoFactorModal.tsx` and `apps/web/components/security/TwoFactorAuthAPI.ts` were not migrated to the new backup-code disable flow and still use the old two-argument `TwoFactorAuthAPI.disable(password, totpCode)` call, so users accessing 2FA disable through the security settings page cannot send a backup code and cannot use backup codes to disable 2FA."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Backup code validation is case-sensitive due to the use of indexOf(). This causes validation to fail if a user enters uppercase hex characters, as backup codes should be case-insensitive for a better user experience.",
+          "severity": "Medium"
+        },
+        {
+          "golden_comment": "Because backupCodes are decrypted and mutated in memory before being written back, two concurrent login requests using the same backupCode could both pass this check and update, so a single backup code may effectively be accepted more than once if used concurrently, weakening the intended one-time-use semantics.",
+          "severity": "High"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 12,
+      "total_scored_candidates": 12,
+      "total_golden": 4,
+      "total_ignored_golden": 0,
+      "tp": 2,
+      "fp": 10,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.16666666666666666,
+      "recall": 0.5,
+      "tool": "deputydev-v1",
+      "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR10600__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR10600__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/calcom/cal.com/pull/10967": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Potential null reference if mainHostDestinationCalendar is undefined if evt.destinationCalendar is null or an empty array ",
+          "severity": "High",
+          "matched_candidate": "When `evt.destinationCalendar` is null, undefined, or empty, `mainHostDestinationCalendar` is undefined and accessing `.integration` crashes Google Meet booking creation before the Daily fallback can run.",
+          "confidence": 0.99,
+          "reasoning": "The candidate describes the same null/undefined dereference: if evt.destinationCalendar is null/undefined/empty, mainHostDestinationCalendar becomes undefined and accessing its properties such as .integration can crash."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "When `externalCalendarId` is falsy, the Google Calendar fallback searches using the absent `externalCalendarId` instead of selecting the event's destination calendar, so `updateEvent` can pass an undefined calendar ID to Google and deletes can fall back to `primary` rather than the stored destination calendar."
+        },
+        {
+          "candidate": "If a calendar reference credential is missing from `this.calendarCredentials` and the database lookup does not return a usable credential, `credential` remains undefined but `updateEvent` is still called, causing the update path to fail when it accesses credential data."
+        },
+        {
+          "candidate": "Creating an event for multiple destination calendars with credentials missing from `this.calendarCredentials` awaits `findUnique` inside the loop, causing N uncached host credentials to incur N sequential database round trips."
+        },
+        {
+          "candidate": "Updating calendar references with credentials missing from `this.calendarCredentials` awaits `findUnique` inside the loop, so a collective reschedule with N uncached credentials incurs N sequential database round trips."
+        },
+        {
+          "candidate": "Cancelling a collective booking awaits a credential lookup inside the loop over calendar references, so every credential missing from `bookingToDelete.user.credentials` adds one sequential database round trip and increases cancellation latency."
+        },
+        {
+          "candidate": "When neither the event type nor organizer has a destination calendar, `evt.destinationCalendar` is initialized to null and optional chaining skips appending team destination calendars, causing collective events to silently omit all team-host destination calendars."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The optional chaining on mainHostDestinationCalendar?.integration is redundant since you already check mainHostDestinationCalendar in the ternary condition.",
+          "severity": "Low"
+        },
+        {
+          "golden_comment": "Logic error: when externalCalendarId is provided, you're searching for a calendar where externalId === externalCalendarId, but this will always fail since you're looking for a calendar that matches itself. Should likely find by credentialId or use different logic.",
+          "severity": "High"
+        },
+        {
+          "golden_comment": "Logic inversion in organization creation: The slug property is now conditionally set when IS_TEAM_BILLING_ENABLED is true, instead of when it's false as originally intended. This change, combined with requestedSlug still being set when IS_TEAM_BILLING_ENABLED is true, results in both properties being set when billing is enabled, and neither when disabled",
+          "severity": "Medium"
+        },
+        {
+          "golden_comment": "The Calendar interface now requires createEvent(event, credentialId), but some implementations (e.g., Lark/Office365) still declare createEvent(event) only—this breaks the interface contract (also applies to other locations in the PR).",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 7,
+      "total_scored_candidates": 7,
+      "total_golden": 5,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 6,
+      "fn": 4,
+      "errors_count": 0,
+      "precision": 0.14285714285714285,
+      "recall": 0.2,
+      "tool": "deputydev-v1",
+      "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR10967__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR10967__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  },
+  "https://github.com/calcom/cal.com/pull/8087": {
+    "deputydev-v1": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The code uses forEach with async callbacks, which causes asynchronous operations (e.g., calendar/video event deletions, payment refunds) to run concurrently without being awaited. This 'fire-and-forget' behavior leads to unhandled promise rejections, race conditions, and incomplete cleanup, as surrounding try-catch blocks cannot properly handle errors from these unawaited promises. Replace forEach with for...of loops or Promise.all() with map() to ensure proper sequential execution and error handling.",
+          "severity": "Critical",
+          "matched_candidate": "In packages/features/bookings/lib/handleCancelBooking.ts at line 460, an async `forEach` for recurring calendar cleanup is not awaited, so callbacks can pause at `getCalendar` while downstream `Promise.all(apiDeletes)` runs before those callbacks have pushed their deletion promises; cancellation can proceed without waiting for recurring calendar deletions, and failures from those deletions are not observed in the handler.",
+          "confidence": 0.96,
+          "reasoning": "The candidate identifies the same core problem: an async callback passed to forEach is not awaited, causing cleanup deletion promises to be missed or run fire-and-forget, with errors not properly observed by the handler. It focuses on one specific occurrence in recurring calendar cleanup, which is within the golden issue's scope."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "In packages/app-store/wipemycalother/lib/reschedule.ts at line 125, `forEach` is used with an async callback for calendar/video deletions, so the deletion promises are not awaited; `Reschedule` can proceed or complete before cleanup finishes, and rejected callback promises bypass the surrounding `try/catch` and remain unobserved."
+        },
+        {
+          "candidate": "In packages/core/videoClient.ts at line 24, awaiting each app module resolution inside a `for...of` loop serializes independent video credential processing; users with multiple video credentials incur the sum of all module-resolution delays instead of resolving them concurrently."
+        },
+        {
+          "candidate": "In packages/trpc/server/routers/viewer/bookings.tsx at line 553, `forEach` discards async callback promises for calendar/video deletion, so the mutation can send the reschedule email and return success before old integration events are removed, and rejected deletion promises are not propagated through the mutation."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Consider adding try-catch around the await to handle import failures gracefully",
+          "severity": "Low"
+        }
+      ],
+      "neutral_matches": [],
+      "ignored_false_negatives": [],
+      "errors": [],
+      "total_candidates": 5,
+      "total_scored_candidates": 4,
+      "total_golden": 2,
+      "total_ignored_golden": 0,
+      "tp": 1,
+      "fp": 3,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.25,
+      "recall": 0.5,
+      "tool": "deputydev-v1",
+      "repo_name": "cal_dot_com__cal.com__deputydev-v5__PR8087__20260819",
+      "pr_url": "https://github.com/dd-benchmarking/cal_dot_com__cal.com__deputydev-v5__PR8087__20260819/pull/1",
+      "golden_comments_source": "embedded",
+      "ignored_severities": []
+    }
+  }
+}
diff --git a/offline/tests/test_step1.py b/offline/tests/test_step1.py
index e877a45..24dc898 100644
--- a/offline/tests/test_step1.py
+++ b/offline/tests/test_step1.py
@@ -86,6 +86,32 @@ def fake_gh(_args):
     ]
 
 
+def test_deputydev_v1_collects_human_account_reviews(monkeypatch):
+    deputydev_user = {"login": "deputydev-reviewer", "type": "User"}
+    responses = [
+        [
+            {
+                "path": "file.py",
+                "line": 10,
+                "body": "finding",
+                "created_at": "2024-01-01",
+                "user": deputydev_user,
+            }
+        ],
+        [],
+        [],
+    ]
+
+    monkeypatch.setattr(step1, "gh", lambda _args: responses.pop(0))
+
+    comments = step1.fetch_review_comments("org", "repo", 1, tool="deputydev-v1")
+
+    assert "deputydev-v1" in step1._NON_BOT_TOOLS
+    assert comments == [
+        {"path": "file.py", "line": 10, "body": "finding", "created_at": "2024-01-01"}
+    ]
+
+
 def test_fetch_repo_data(monkeypatch):
     def stub_pr_metadata(_org, _repo, pr):
         return {"title": f"PR {pr}", "url": "https://github.com/org/repo/pull/1"}