From 064f6f68a2db7c413e59a38a943c9b081c4e85fb Mon Sep 17 00:00:00 2001 From: taisazero Date: Mon, 13 Apr 2026 14:58:10 -0700 Subject: [PATCH 1/6] let there be light --- .../code_review_benchmark/parsers/__init__.py | 34 ++ offline/code_review_benchmark/parsers/base.py | 68 ++++ .../parsers/coderabbit.py | 364 ++++++++++++++++++ .../code_review_benchmark/parsers/default.py | 25 ++ .../step1_5_parse_reviews.py | 268 +++++++++++++ offline/pyproject.toml | 1 + offline/tests/test_coderabbit_classifier.py | 98 +++++ offline/tests/test_coderabbit_inline.py | 90 +++++ offline/tests/test_coderabbit_parser.py | 115 ++++++ offline/tests/test_coderabbit_status.py | 153 ++++++++ offline/tests/test_details_splitter.py | 69 ++++ offline/tests/test_integration_coderabbit.py | 90 +++++ offline/tests/test_parsers_base.py | 141 +++++++ offline/tests/test_parsers_default.py | 35 ++ offline/tests/test_parsers_registry.py | 18 + offline/tests/test_step1_5.py | 136 +++++++ offline/tests/test_step2.py | 59 +++ 17 files changed, 1764 insertions(+) create mode 100644 offline/code_review_benchmark/parsers/__init__.py create mode 100644 offline/code_review_benchmark/parsers/base.py create mode 100644 offline/code_review_benchmark/parsers/coderabbit.py create mode 100644 offline/code_review_benchmark/parsers/default.py create mode 100644 offline/code_review_benchmark/step1_5_parse_reviews.py create mode 100644 offline/tests/test_coderabbit_classifier.py create mode 100644 offline/tests/test_coderabbit_inline.py create mode 100644 offline/tests/test_coderabbit_parser.py create mode 100644 offline/tests/test_coderabbit_status.py create mode 100644 offline/tests/test_details_splitter.py create mode 100644 offline/tests/test_integration_coderabbit.py create mode 100644 offline/tests/test_parsers_base.py create mode 100644 offline/tests/test_parsers_default.py create mode 100644 offline/tests/test_parsers_registry.py create mode 100644 offline/tests/test_step1_5.py diff --git a/offline/code_review_benchmark/parsers/__init__.py b/offline/code_review_benchmark/parsers/__init__.py new file mode 100644 index 0000000..e69b1a4 --- /dev/null +++ b/offline/code_review_benchmark/parsers/__init__.py @@ -0,0 +1,34 @@ +"""Pluggable parser framework for tool-specific comment parsing.""" + +from __future__ import annotations + +from code_review_benchmark.parsers.base import BaseParser +from code_review_benchmark.parsers.base import ParsedComment +from code_review_benchmark.parsers.base import ParsedReview + +__all__ = ["BaseParser", "ParsedComment", "ParsedReview", "get_parser"] + +# Registry — populated lazily on first get_parser() call +PARSERS: dict[str, type[BaseParser]] = {} +_registered = False + + +def _register_parsers() -> None: + global _registered + if _registered: + return + _registered = True + try: + from code_review_benchmark.parsers.coderabbit import CodeRabbitParser + + PARSERS["coderabbit"] = CodeRabbitParser + except ImportError: + pass # CodeRabbitParser not yet created + + +def get_parser(tool: str) -> BaseParser: + """Return the parser for a given tool, or DefaultParser if none registered.""" + from code_review_benchmark.parsers.default import DefaultParser + + _register_parsers() + return PARSERS.get(tool, DefaultParser)() diff --git a/offline/code_review_benchmark/parsers/base.py b/offline/code_review_benchmark/parsers/base.py new file mode 100644 index 0000000..6b6ce09 --- /dev/null +++ b/offline/code_review_benchmark/parsers/base.py @@ -0,0 +1,68 @@ +"""Base classes for the parser framework.""" + +from __future__ import annotations + +from abc import ABC +from abc import abstractmethod +from dataclasses import dataclass +from dataclasses import field + + +@dataclass +class ParsedComment: + """A single parsed comment with section classification and metadata.""" + + body: str + path: str | None + line: int | None + created_at: str + section: str + severity: str | None + + def to_dict(self) -> dict: + return { + "body": self.body, + "path": self.path, + "line": self.line, + "created_at": self.created_at, + "section": self.section, + "severity": self.severity, + } + + +@dataclass +class ParsedReview: + """A parsed review containing comments categorized by section.""" + + sections: dict[str, list[ParsedComment]] = field(default_factory=dict) + + def to_markdown(self) -> str: + """Render all comments as clean markdown for LLM consumption.""" + parts: list[str] = [] + for _section_name, comments in self.sections.items(): + for comment in comments: + header = _format_header(comment) + parts.append(f"{header}\n\n{comment.body}") + return "\n\n---\n\n".join(parts) + + +def _format_header(comment: ParsedComment) -> str: + """Format a markdown header for a comment.""" + section_label = comment.section.replace("_", " ").title() + if comment.path and comment.line is not None: + location = f"{comment.path}:{comment.line}" + if comment.severity: + return f"## {section_label} — {location} [{comment.severity}]" + return f"## {section_label} — {location}" + if comment.severity: + return f"## {section_label} [{comment.severity}]" + return f"## {section_label}" + + +class BaseParser(ABC): + """Abstract base for tool-specific comment parsers.""" + + @abstractmethod + def parse(self, review_comments: list[dict]) -> ParsedReview: + """Parse raw review comments into categorized sections.""" + ... diff --git a/offline/code_review_benchmark/parsers/coderabbit.py b/offline/code_review_benchmark/parsers/coderabbit.py new file mode 100644 index 0000000..50b269c --- /dev/null +++ b/offline/code_review_benchmark/parsers/coderabbit.py @@ -0,0 +1,364 @@ +"""CodeRabbit comment parser. + +Parses CodeRabbit's structured markdown/HTML comments into categorized sections. +Uses marker-based classification for comment-level types and html-based parsing +for nested
blocks within status comments. +""" + +from __future__ import annotations + +import re + +from code_review_benchmark.parsers.base import BaseParser +from code_review_benchmark.parsers.base import ParsedComment +from code_review_benchmark.parsers.base import ParsedReview + +# --------------------------------------------------------------------------- +# Severity badge patterns +# --------------------------------------------------------------------------- + +_SEVERITY_PATTERNS = { + "critical": re.compile(r"_\U0001f534\s*Critical_", re.IGNORECASE), + "major": re.compile(r"_\U0001f7e0\s*Major_", re.IGNORECASE), + "minor": re.compile(r"_\U0001f7e1\s*Minor_", re.IGNORECASE), +} + +_SEVERITY_BADGE_LINE = re.compile( + r"^_\u26a0\ufe0f\s*Potential issue_(\s*\|\s*_[^\n_]*_)?\s*\n*", + re.MULTILINE, +) + +# --------------------------------------------------------------------------- +# Noise patterns +# --------------------------------------------------------------------------- + +_HTML_COMMENT = re.compile(r"", re.DOTALL) +_DETAILS_BLOCK_RE = re.compile( + r"
\s*(.*?).*?
", + re.DOTALL, +) +_NOISE_SUMMARIES = { + "\U0001f9e9 Analysis chain", + "\U0001f916 Prompt for AI Agents", + "\U0001f9f0 Tools", # Wrapper for linting tool output (RuboCop, Ruff, Biome, Brakeman) + "\U0001fa9b", # 🪛 Individual linting tool blocks (standalone) +} + +# --------------------------------------------------------------------------- +# Status comment section mapping +# --------------------------------------------------------------------------- + +_SECTION_SUMMARY_MAP = { + "\U0001f916 Fix all issues": "actionable_summary", + "\U0001f9f9 Nitpick comments": "nitpick", + "\u26a0\ufe0f Outside diff range comments": "outside_diff", + "\U0001f4dc Review details": "review_details", + "\U0001f509 Additional comments": "additional_comments", + "\U0001f9f0 Additional context used": "additional_context", +} + +_SUMMARY_RE = re.compile(r"(.*?)", re.DOTALL) +_LINE_RANGE_RE = re.compile(r"^`(\d+)(?:-\d+)?`:\s*", re.MULTILINE) +_FILE_SUMMARY_RE = re.compile(r"^(.*?)\s*\(\d+\)$") + + +# --------------------------------------------------------------------------- +# Comment-level classification +# --------------------------------------------------------------------------- + + +def classify_comment(comment: dict) -> str: + """Classify a raw comment into a section type based on markers. + + Checks run in priority order: inline first, then HTML comment markers, + then content-based detection. + """ + path = comment.get("path") + line = comment.get("line") + body = comment.get("body", "") + + # Inline comments: path and line both set + if path and line is not None: + return "inline" + + # HTML comment markers (most reliable) + if "" in body: + return "walkthrough" + + # Status comment: starts with actionable comments count + if body.lstrip().startswith("**Actionable comments posted:"): + return "status" + + # Standalone outside-diff (not inside a status comment) + if "\u26a0\ufe0f Outside diff range comments" in body: + return "outside_diff" + + if "" in body: + return "pre_merge_checks" + + if "" in body: + return "finishing_touches" + + return "unknown" + + +# --------------------------------------------------------------------------- +#
block splitter +# --------------------------------------------------------------------------- + +_DETAILS_TAG_RE = re.compile(r"]*)?>|
", re.IGNORECASE) + + +def split_details_blocks(html: str) -> list[str]: + """Split HTML into top-level
...
blocks. + + Returns the full text of each top-level block including nested blocks. + """ + blocks: list[str] = [] + depth = 0 + start = 0 + + for match in _DETAILS_TAG_RE.finditer(html): + tag_text = match.group() + if tag_text.lower().startswith("": + if depth == 0: + # Orphaned closing tag — skip to avoid going negative + continue + depth -= 1 + if depth == 0: + blocks.append(html[start : match.end()]) + return blocks + + +# --------------------------------------------------------------------------- +# Severity parsing and inline noise removal +# --------------------------------------------------------------------------- + + +def parse_severity(body: str) -> str | None: + """Extract severity level from inline comment badge patterns.""" + for severity, pattern in _SEVERITY_PATTERNS.items(): + if pattern.search(body): + return severity + return None + + +def clean_inline_body(body: str) -> str: + """Remove noise from an inline comment body, preserving issue text and code suggestions.""" + # Remove severity badge line + cleaned = _SEVERITY_BADGE_LINE.sub("", body) + + # Remove noisy
blocks using proper nesting-aware splitter + # (handles nested blocks like 🧰 Tools containing 🪛 RuboCop/Ruff/Biome) + blocks = split_details_blocks(cleaned) + for block in reversed(blocks): + summary_match = _SUMMARY_RE.search(block[:200]) + if summary_match: + summary_text = summary_match.group(1).strip() + if any(noise in summary_text for noise in _NOISE_SUMMARIES): + cleaned = cleaned.replace(block, "") + + # Remove HTML comments (internal state, fingerprinting, etc.) + cleaned = _HTML_COMMENT.sub("", cleaned) + + return cleaned.strip() + + +# --------------------------------------------------------------------------- +# Status comment section parser +# --------------------------------------------------------------------------- + + +def _classify_details_block(block: str) -> str | None: + """Identify which section a top-level
block belongs to.""" + summary_match = _SUMMARY_RE.search(block[:300]) + if not summary_match: + return None + summary_text = summary_match.group(1).strip() + for pattern, section_name in _SECTION_SUMMARY_MAP.items(): + if pattern in summary_text: + return section_name + return None + + +def _parse_file_grouped_section(block: str) -> list[dict]: + """Parse a section that groups comments by file (nitpick, outside_diff). + + Returns list of dicts with body, path, line keys. + """ + results: list[dict] = [] + # Get inner content after the outer summary + outer_summary_end = block.find("") + if outer_summary_end == -1: + return results + inner_content = block[outer_summary_end + len("") :] + # Remove the closing
of the outer block + last_close = inner_content.rfind("
") + if last_close != -1: + inner_content = inner_content[:last_close] + + file_blocks = split_details_blocks(inner_content) + + for file_block in file_blocks: + # Extract file path from summary + summary_match = _SUMMARY_RE.search(file_block[:300]) + if not summary_match: + continue + summary_text = summary_match.group(1).strip() + # Remove count suffix: "path/to/file.py (2)" -> "path/to/file.py" + file_match = _FILE_SUMMARY_RE.match(summary_text) + file_path = file_match.group(1).strip() if file_match else summary_text + + # Extract content after summary, before closing tags + content_start = file_block.find("") + if content_start == -1: + continue + content = file_block[content_start + len("") :] + # Strip blockquote tags + content = re.sub(r"", "", content) + # Strip trailing
+ content_last_close = content.rfind("") + if content_last_close != -1: + content = content[:content_last_close] + # Strip markdown blockquote prefixes (> ) from each line + content = re.sub(r"^>\s?", "", content, flags=re.MULTILINE) + content = content.strip() + + if not content: + continue + + # Split individual comments by --- delimiter + items = re.split(r"\n---\n", content) + for item in items: + item = item.strip() + if not item: + continue + # Extract line number from `NNN-NNN`: pattern + line_match = _LINE_RANGE_RE.search(item) + line_num = int(line_match.group(1)) if line_match else None + + results.append({ + "body": item, + "path": file_path, + "line": line_num, + }) + + return results + + +def parse_status_comment(body: str) -> dict[str, list[dict]]: + """Parse a status comment into its sub-sections. + + Returns dict mapping section names to lists of comment dicts. + """ + sections: dict[str, list[dict]] = {} + + # Split into top-level
blocks + blocks = split_details_blocks(body) + + for block in blocks: + section_name = _classify_details_block(block) + if section_name is None: + continue + + if section_name in ("nitpick", "outside_diff"): + # These have file-grouped inner structure + items = _parse_file_grouped_section(block) + if items: + sections.setdefault(section_name, []).extend(items) + else: + # Other sections: extract the content between summary and closing + summary_end = block.find("") + if summary_end == -1: + continue + content = block[summary_end + len("") :] + last_close = content.rfind("
") + if last_close != -1: + content = content[:last_close] + content = re.sub(r"", "", content).strip() + sections[section_name] = [{"body": content, "path": None, "line": None}] + + return sections + + +# --------------------------------------------------------------------------- +# Full parser +# --------------------------------------------------------------------------- + + +class CodeRabbitParser(BaseParser): + """Parses CodeRabbit structured comments into categorized sections.""" + + def parse(self, review_comments: list[dict]) -> ParsedReview: + sections: dict[str, list[ParsedComment]] = {} + + for comment in review_comments: + comment_type = classify_comment(comment) + body = comment.get("body", "") + path = comment.get("path") + line = comment.get("line") + created_at = comment.get("created_at", "") + + if comment_type == "inline": + severity = parse_severity(body) + cleaned_body = clean_inline_body(body) + parsed = ParsedComment( + body=cleaned_body, + path=path, + line=line, + created_at=created_at, + section="inline", + severity=severity, + ) + sections.setdefault("inline", []).append(parsed) + + elif comment_type == "status": + sub_sections = parse_status_comment(body) + for section_name, items in sub_sections.items(): + for item in items: + parsed = ParsedComment( + body=item["body"], + path=item.get("path"), + line=item.get("line"), + created_at=created_at, + section=section_name, + severity=None, + ) + sections.setdefault(section_name, []).append(parsed) + + elif comment_type == "outside_diff": + # Standalone outside-diff comment — parse like a status section + sub_sections = parse_status_comment(body) + if "outside_diff" in sub_sections: + for item in sub_sections["outside_diff"]: + parsed = ParsedComment( + body=item["body"], + path=item.get("path"), + line=item.get("line"), + created_at=created_at, + section="outside_diff", + severity=None, + ) + sections.setdefault("outside_diff", []).append(parsed) + else: + cleaned = _HTML_COMMENT.sub("", body).strip() + parsed = ParsedComment( + body=cleaned, path=None, line=None, + created_at=created_at, section="outside_diff", severity=None, + ) + sections.setdefault("outside_diff", []).append(parsed) + + else: + # walkthrough, pre_merge_checks, finishing_touches, unknown + cleaned = _HTML_COMMENT.sub("", body).strip() + parsed = ParsedComment( + body=cleaned, path=path, line=line, + created_at=created_at, section=comment_type, severity=None, + ) + sections.setdefault(comment_type, []).append(parsed) + + return ParsedReview(sections=sections) diff --git a/offline/code_review_benchmark/parsers/default.py b/offline/code_review_benchmark/parsers/default.py new file mode 100644 index 0000000..e3b3d07 --- /dev/null +++ b/offline/code_review_benchmark/parsers/default.py @@ -0,0 +1,25 @@ +"""Default passthrough parser for tools without custom parsing.""" + +from __future__ import annotations + +from code_review_benchmark.parsers.base import BaseParser +from code_review_benchmark.parsers.base import ParsedComment +from code_review_benchmark.parsers.base import ParsedReview + + +class DefaultParser(BaseParser): + """Passes all comments through as a single 'raw' section.""" + + def parse(self, review_comments: list[dict]) -> ParsedReview: + comments = [ + ParsedComment( + body=c.get("body", ""), + path=c.get("path"), + line=c.get("line"), + created_at=c.get("created_at", ""), + section="raw", + severity=None, + ) + for c in review_comments + ] + return ParsedReview(sections={"raw": comments}) diff --git a/offline/code_review_benchmark/step1_5_parse_reviews.py b/offline/code_review_benchmark/step1_5_parse_reviews.py new file mode 100644 index 0000000..1aee378 --- /dev/null +++ b/offline/code_review_benchmark/step1_5_parse_reviews.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""Parse tool review comments into categorized sections. + +Produces results/parsed_{tool}.json with filtered comments and rendered markdown. +Step2 auto-detects these files and uses them instead of raw comments. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from code_review_benchmark.parsers import get_parser +from code_review_benchmark.parsers.base import ParsedReview + +RESULTS_DIR = Path("results") +BENCHMARK_DATA_FILE = RESULTS_DIR / "benchmark_data.json" + +DEFAULT_SECTIONS = { + "inline": True, + "actionable_summary": True, + "outside_diff": True, + "nitpick": False, + "walkthrough": False, + "pre_merge_checks": False, + "finishing_touches": False, + "review_details": False, + "additional_comments": False, + "additional_context": False, + "unknown": False, +} + + +def resolve_sections( + include: list[str] | None, + exclude: list[str] | None, + only: list[str] | None, +) -> dict[str, bool]: + """Resolve section filter config from CLI flags.""" + if only: + return {name: (name in only) for name in DEFAULT_SECTIONS} + config = dict(DEFAULT_SECTIONS) + if include: + for name in include: + if name in config: + config[name] = True + if exclude: + for name in exclude: + if name in config: + config[name] = False + return config + + +SEVERITY_RANK = {"critical": 3, "major": 2, "minor": 1} + + +def build_review_output( + tool: str, + review_comments: list[dict], + sections_config: dict[str, bool], + min_severity: str | None = None, +) -> dict: + """Parse and filter a single review's comments, returning the output structure.""" + parser = get_parser(tool) + parsed: ParsedReview = parser.parse(review_comments) + + min_rank = SEVERITY_RANK.get(min_severity, 0) if min_severity else 0 + + included_comments: list[dict] = [] + excluded_comments: list[dict] = [] + + # Build a filtered ParsedReview for markdown rendering + included_sections: dict = {} + + for section_name, comments in parsed.sections.items(): + section_included = sections_config.get(section_name, False) + for comment in comments: + d = comment.to_dict() + # Filter by severity if min_severity is set + comment_included = section_included + if comment_included and min_rank > 0 and comment.severity: + if SEVERITY_RANK.get(comment.severity, 0) < min_rank: + comment_included = False + if comment_included: + included_comments.append(d) + included_sections.setdefault(section_name, []).append(comment) + else: + excluded_comments.append(d) + + # Render markdown from included sections only + included_review = ParsedReview(sections=included_sections) + rendered_markdown = included_review.to_markdown() + + return { + "tool": tool, + "review_comments": included_comments, + "excluded_comments": excluded_comments, + "rendered_markdown": rendered_markdown, + } + + +def run_parser(tool: str, sections_config: dict[str, bool], min_severity: str | None = None) -> None: + """Run the parser for a specific tool and write results.""" + if not BENCHMARK_DATA_FILE.exists(): + print(f"Error: {BENCHMARK_DATA_FILE} not found") + return + + with open(BENCHMARK_DATA_FILE) as f: + data = json.load(f) + + included = [k for k, v in sections_config.items() if v] + excluded = [k for k, v in sections_config.items() if not v] + + output = { + "config": { + "tool": tool, + "included_sections": included, + "excluded_sections": excluded, + "min_severity": min_severity, + }, + "reviews": {}, + } + + review_count = 0 + for golden_url, entry in data.items(): + for review in entry.get("reviews", []): + if review["tool"] != tool: + continue + comments = review.get("review_comments", []) + review_output = build_review_output(tool, comments, sections_config, min_severity) + output["reviews"][golden_url] = review_output + review_count += 1 + + output_file = RESULTS_DIR / f"parsed_{tool}.json" + with open(output_file, "w") as f: + json.dump(output, f, indent=2) + + print(f"Parsed {review_count} {tool} reviews") + print(f"Included sections: {included}") + if min_severity: + print(f"Min severity: {min_severity}") + print(f"Output: {output_file}") + + +def _format_candidate_text(body: str, path: str | None, line: int | None, include_path: bool) -> str: + """Format candidate text, optionally prefixing with file path.""" + if include_path and path: + location = f"{path}:{line}" if line is not None else path + return f"{location} — {body}" + return body + + +def write_candidates_direct( + tool: str, + sections_config: dict[str, bool], + min_severity: str | None = None, + model_dir: str | None = None, + include_path: bool = False, +) -> None: + """Write parsed comments directly to candidates.json, bypassing step2 LLM extraction. + + Each parsed comment becomes a candidate entry with text, path, line, and source fields. + """ + if not BENCHMARK_DATA_FILE.exists(): + print(f"Error: {BENCHMARK_DATA_FILE} not found") + return + + with open(BENCHMARK_DATA_FILE) as f: + data = json.load(f) + + # Determine output directory + import os + + if model_dir: + out_dir = RESULTS_DIR / model_dir + else: + model = os.environ.get("MARTIAN_MODEL", "direct") + out_dir = RESULTS_DIR / model.replace("/", "_") + out_dir.mkdir(parents=True, exist_ok=True) + + # Load existing candidates (preserve other tools) + candidates_file = out_dir / "candidates.json" + if candidates_file.exists(): + with open(candidates_file) as f: + all_candidates = json.load(f) + else: + all_candidates = {} + + review_count = 0 + total_candidates = 0 + + for golden_url, entry in data.items(): + for review in entry.get("reviews", []): + if review["tool"] != tool: + continue + comments = review.get("review_comments", []) + review_output = build_review_output(tool, comments, sections_config, min_severity) + + # Convert included comments to candidate format + candidates = [] + for c in review_output["review_comments"]: + text = _format_candidate_text(c["body"], c.get("path"), c.get("line"), include_path) + candidates.append({ + "text": text, + "path": c.get("path"), + "line": c.get("line"), + "source": "parsed", + }) + + if golden_url not in all_candidates: + all_candidates[golden_url] = {} + all_candidates[golden_url][tool] = candidates + review_count += 1 + total_candidates += len(candidates) + + with open(candidates_file, "w") as f: + json.dump(all_candidates, f, indent=2) + + included = [k for k, v in sections_config.items() if v] + print(f"Direct pass-through: {review_count} {tool} reviews → {total_candidates} candidates") + print(f"Included sections: {included}") + if min_severity: + print(f"Min severity: {min_severity}") + print(f"Output: {candidates_file}") + + +def main(): + parser = argparse.ArgumentParser(description="Parse tool review comments into sections") + parser.add_argument("--tool", default="coderabbit", help="Tool to parse (default: coderabbit)") + parser.add_argument("--include", nargs="*", help="Additional sections to include") + parser.add_argument("--exclude", nargs="*", help="Sections to exclude from defaults") + parser.add_argument("--only", help="Comma-separated list of sections to include (exclusive)") + parser.add_argument("--preview", action="store_true", help="Print rendered markdown to stdout") + parser.add_argument("--min-severity", choices=["minor", "major", "critical"], help="Exclude inline comments below this severity") + parser.add_argument("--write-candidates", action="store_true", help="Write parsed comments directly to candidates.json (bypass step2)") + parser.add_argument("--model-dir", help="Model directory name for candidates.json output (used with --write-candidates)") + parser.add_argument("--include-path", action="store_true", help="Prefix candidate text with file path (used with --write-candidates)") + args = parser.parse_args() + + only = args.only.split(",") if args.only else None + sections_config = resolve_sections(args.include, args.exclude, only) + + if args.preview: + if not BENCHMARK_DATA_FILE.exists(): + print(f"Error: {BENCHMARK_DATA_FILE} not found") + return + with open(BENCHMARK_DATA_FILE) as f: + data = json.load(f) + for golden_url, entry in data.items(): + for review in entry.get("reviews", []): + if review["tool"] != args.tool: + continue + result = build_review_output(args.tool, review.get("review_comments", []), sections_config, args.min_severity) + print(f"\n{'=' * 60}") + print(f"PR: {golden_url}") + print(f"{'=' * 60}") + print(result["rendered_markdown"]) + return + + if args.write_candidates: + write_candidates_direct(args.tool, sections_config, args.min_severity, args.model_dir, args.include_path) + else: + run_parser(args.tool, sections_config, args.min_severity) + + +if __name__ == "__main__": + main() diff --git a/offline/pyproject.toml b/offline/pyproject.toml index 2087167..7d161e9 100644 --- a/offline/pyproject.toml +++ b/offline/pyproject.toml @@ -19,6 +19,7 @@ dev = [ [project.scripts] fork-prs = "code_review_benchmark.step0_fork_prs:main" download-prs = "code_review_benchmark.step1_download_prs:main" +parse-reviews = "code_review_benchmark.step1_5_parse_reviews:main" extract-comments = "code_review_benchmark.step2_extract_comments:main" judge-comments = "code_review_benchmark.step3_judge_comments:main" export-by-tool = "code_review_benchmark.step4_export_by_tool:main" diff --git a/offline/tests/test_coderabbit_classifier.py b/offline/tests/test_coderabbit_classifier.py new file mode 100644 index 0000000..7dcafa4 --- /dev/null +++ b/offline/tests/test_coderabbit_classifier.py @@ -0,0 +1,98 @@ +"""Tests for CodeRabbit comment-level classification.""" + +from __future__ import annotations + +from code_review_benchmark.parsers.coderabbit import classify_comment + + +def test_inline_comment(): + comment = {"body": "Some issue here", "path": "src/main.py", "line": 42} + assert classify_comment(comment) == "inline" + + +def test_inline_requires_both_path_and_line(): + comment = {"body": "Some issue", "path": "src/main.py", "line": None} + assert classify_comment(comment) != "inline" + + comment2 = {"body": "Some issue", "path": None, "line": 42} + assert classify_comment(comment2) != "inline" + + +def test_walkthrough_comment(): + comment = { + "body": ( + "\n\n
\n" + "\U0001f4dd Walkthrough\n\n## Walkthrough\nSome changes..." + ), + "path": None, + "line": None, + } + assert classify_comment(comment) == "walkthrough" + + +def test_status_comment(): + comment = { + "body": "**Actionable comments posted: 4**\n\n
\n\U0001f916 Fix all issues", + "path": None, + "line": None, + } + assert classify_comment(comment) == "status" + + +def test_outside_diff_standalone(): + comment = { + "body": ( + "> [!CAUTION]\n> Some comments are outside the diff\n\n" + "
\n\u26a0\ufe0f Outside diff range comments (2)" + ), + "path": None, + "line": None, + } + assert classify_comment(comment) == "outside_diff" + + +def test_pre_merge_checks(): + comment = { + "body": "\n\n
\n\U0001f6a5 Pre-merge checks", + "path": None, + "line": None, + } + assert classify_comment(comment) == "pre_merge_checks" + + +def test_finishing_touches(): + comment = { + "body": "\n\n
\n\u2728 Finishing touches", + "path": None, + "line": None, + } + assert classify_comment(comment) == "finishing_touches" + + +def test_unknown_comment(): + comment = { + "body": "This is just a regular comment with no markers.", + "path": None, + "line": None, + } + assert classify_comment(comment) == "unknown" + + +def test_composite_comment_walkthrough_wins(): + """When a comment contains walkthrough + pre_merge + finishing, walkthrough wins (checked first).""" + comment = { + "body": "\nstuff\n\nmore stuff", + "path": None, + "line": None, + } + assert classify_comment(comment) == "walkthrough" + + +def test_inline_takes_priority_over_markers(): + """If path+line are set, it's inline even if body has walkthrough markers.""" + comment = { + "body": "\nSome text", + "path": "src/main.py", + "line": 10, + } + assert classify_comment(comment) == "inline" diff --git a/offline/tests/test_coderabbit_inline.py b/offline/tests/test_coderabbit_inline.py new file mode 100644 index 0000000..e271aa2 --- /dev/null +++ b/offline/tests/test_coderabbit_inline.py @@ -0,0 +1,90 @@ +"""Tests for inline comment severity parsing and noise removal.""" + +from __future__ import annotations + +from code_review_benchmark.parsers.coderabbit import clean_inline_body +from code_review_benchmark.parsers.coderabbit import parse_severity + + +def test_parse_severity_critical(): + body = "_\u26a0\ufe0f Potential issue_ | _\U0001f534 Critical_\n\nSome issue text" + assert parse_severity(body) == "critical" + + +def test_parse_severity_major(): + body = "_\u26a0\ufe0f Potential issue_ | _\U0001f7e0 Major_\n\nSome issue text" + assert parse_severity(body) == "major" + + +def test_parse_severity_minor(): + body = "_\u26a0\ufe0f Potential issue_ | _\U0001f7e1 Minor_\n\nSome issue text" + assert parse_severity(body) == "minor" + + +def test_parse_severity_none(): + body = "Just a regular comment without severity badges" + assert parse_severity(body) is None + + +def test_parse_severity_potential_issue_only(): + """If only the type indicator is present without a severity level, return None.""" + body = "_\u26a0\ufe0f Potential issue_\n\nSome text" + assert parse_severity(body) is None + + +def test_clean_inline_removes_analysis_chain(): + body = ( + "_\u26a0\ufe0f Potential issue_ | _\U0001f534 Critical_\n\n" + "
\n\U0001f9e9 Analysis chain\n\n" + "\U0001f3c1 Script executed:\n```shell\nrg -n 'foo'\n```\nLength of output: 500\n\n" + "
\n\n" + "The actual issue is here." + ) + cleaned = clean_inline_body(body) + assert "Analysis chain" not in cleaned + assert "Script executed" not in cleaned + assert "rg -n" not in cleaned + assert "The actual issue is here." in cleaned + + +def test_clean_inline_removes_ai_agent_prompt(): + body = ( + "Some issue text.\n\n" + "
\n\U0001f916 Prompt for AI Agents\n\n" + "Fix the code by doing X\n\n" + "
\n\n" + "" + ) + cleaned = clean_inline_body(body) + assert "Prompt for AI Agents" not in cleaned + assert "fingerprinting" not in cleaned + assert "Some issue text." in cleaned + + +def test_clean_inline_removes_html_comments(): + body = "Issue text\n\nlots of data\n\nMore text" + cleaned = clean_inline_body(body) + assert "internal state" not in cleaned + assert "Issue text" in cleaned + assert "More text" in cleaned + + +def test_clean_inline_preserves_code_suggestions(): + body = "Missing null check.\n\n**Suggested fix:**\n```java\nif (x != null) { ... }\n```" + cleaned = clean_inline_body(body) + assert "Missing null check." in cleaned + assert "```java" in cleaned + assert "if (x != null)" in cleaned + + +def test_clean_inline_removes_severity_badge_line(): + body = "_\u26a0\ufe0f Potential issue_ | _\U0001f534 Critical_\n\nThe actual issue description." + cleaned = clean_inline_body(body) + assert "_\u26a0\ufe0f Potential issue_" not in cleaned + assert "The actual issue description." in cleaned + + +def test_clean_inline_strips_whitespace(): + body = "\n\n Issue text \n\n" + cleaned = clean_inline_body(body) + assert cleaned == "Issue text" diff --git a/offline/tests/test_coderabbit_parser.py b/offline/tests/test_coderabbit_parser.py new file mode 100644 index 0000000..818b74a --- /dev/null +++ b/offline/tests/test_coderabbit_parser.py @@ -0,0 +1,115 @@ +"""Tests for the full CodeRabbitParser.parse() method.""" + +from __future__ import annotations + +from code_review_benchmark.parsers.coderabbit import CodeRabbitParser + + +def test_parse_inline_comment(): + comments = [ + { + "body": "_\u26a0\ufe0f Potential issue_ | _\U0001f534 Critical_\n\nMissing null check.", + "path": "src/main.py", + "line": 42, + "created_at": "2026-01-26T00:00:00Z", + } + ] + parser = CodeRabbitParser() + result = parser.parse(comments) + assert "inline" in result.sections + assert len(result.sections["inline"]) == 1 + c = result.sections["inline"][0] + assert c.severity == "critical" + assert c.path == "src/main.py" + assert c.line == 42 + assert "Missing null check." in c.body + # Severity badge should be stripped from body + assert "_\u26a0\ufe0f Potential issue_" not in c.body + + +def test_parse_walkthrough_classified(): + comments = [ + { + "body": "\n
\U0001f4dd Walkthrough\nStuff\n
", + "path": None, + "line": None, + "created_at": "2026-01-26T00:00:00Z", + } + ] + parser = CodeRabbitParser() + result = parser.parse(comments) + assert "walkthrough" in result.sections + assert len(result.sections["walkthrough"]) == 1 + + +def test_parse_status_comment_splits_sections(): + comments = [ + { + "body": ( + "**Actionable comments posted: 1**\n\n" + "
\n\U0001f916 Fix all issues with AI agents\n\nFix stuff\n\n
\n\n" + "
\n\U0001f9f9 Nitpick comments (1)
\n\n" + "
\na.py (1)
\n\n" + "`10-15`: **Style issue**\n\nUse better names.\n\n" + "
\n\n" + "
" + ), + "path": None, + "line": None, + "created_at": "2026-01-26T00:00:00Z", + } + ] + parser = CodeRabbitParser() + result = parser.parse(comments) + assert "actionable_summary" in result.sections + assert "nitpick" in result.sections + assert result.sections["nitpick"][0].path == "a.py" + + +def test_parse_mixed_comment_types(): + comments = [ + { + "body": "_\u26a0\ufe0f Potential issue_ | _\U0001f7e0 Major_\n\nBug in handler.", + "path": "src/handler.py", + "line": 10, + "created_at": "2026-01-26T00:00:00Z", + }, + { + "body": "\nWalkthrough text", + "path": None, + "line": None, + "created_at": "2026-01-26T00:01:00Z", + }, + { + "body": "Just a random comment.", + "path": None, + "line": None, + "created_at": "2026-01-26T00:02:00Z", + }, + ] + parser = CodeRabbitParser() + result = parser.parse(comments) + assert "inline" in result.sections + assert "walkthrough" in result.sections + assert "unknown" in result.sections + assert result.sections["inline"][0].severity == "major" + + +def test_parse_empty_comments(): + parser = CodeRabbitParser() + result = parser.parse([]) + assert result.sections == {} + + +def test_parse_preserves_created_at(): + comments = [ + { + "body": "Some issue.", + "path": "a.py", + "line": 1, + "created_at": "2026-01-26T16:38:36Z", + } + ] + parser = CodeRabbitParser() + result = parser.parse(comments) + assert result.sections["inline"][0].created_at == "2026-01-26T16:38:36Z" diff --git a/offline/tests/test_coderabbit_status.py b/offline/tests/test_coderabbit_status.py new file mode 100644 index 0000000..04cf0af --- /dev/null +++ b/offline/tests/test_coderabbit_status.py @@ -0,0 +1,153 @@ +"""Tests for status comment section parsing.""" + +from __future__ import annotations + +from code_review_benchmark.parsers.coderabbit import parse_status_comment + + +def test_actionable_summary_extracted(): + body = ( + "**Actionable comments posted: 2**\n\n" + "
\n\U0001f916 Fix all issues with AI agents\n\n" + "```\nIn `@src/main.py`:\n- Around line 42: Missing null check\n```\n\n" + "
\n\n" + "
\n\U0001f4dc Review details\nConfig stuff\n
" + ) + sections = parse_status_comment(body) + assert "actionable_summary" in sections + assert len(sections["actionable_summary"]) == 1 + assert "Missing null check" in sections["actionable_summary"][0]["body"] + + +def test_nitpick_section_extracted(): + body = ( + "**Actionable comments posted: 0**\n\n" + "
\n\U0001f9f9 Nitpick comments (2)
\n\n" + "
\nsrc/util.py (1)
\n\n" + "`10-15`: **Use List.of() instead of Arrays.asList()**\n\n" + "Description of the nitpick.\n\n" + "
\n\n" + "
\nsrc/main.py (1)
\n\n" + "`30-35`: **Rename variable for clarity**\n\n" + "Another nitpick.\n\n" + "
\n\n" + "
" + ) + sections = parse_status_comment(body) + assert "nitpick" in sections + assert len(sections["nitpick"]) == 2 + assert sections["nitpick"][0]["path"] == "src/util.py" + assert sections["nitpick"][1]["path"] == "src/main.py" + + +def test_outside_diff_section_extracted(): + body = ( + "**Actionable comments posted: 1**\n\n" + "> [!CAUTION]\n" + "> Some comments are outside the diff\n\n" + "
\n\u26a0\ufe0f Outside diff range comments (1)
\n\n" + "
\nsrc/init.py (1)
\n\n" + "`59-70`: **Race condition in initialization.**\n\n" + "Details here.\n\n" + "
\n\n" + "
\n\n" + "
\n\U0001f4dc Review details\nStuff\n
" + ) + sections = parse_status_comment(body) + assert "outside_diff" in sections + assert len(sections["outside_diff"]) == 1 + assert sections["outside_diff"][0]["path"] == "src/init.py" + assert "Race condition" in sections["outside_diff"][0]["body"] + + +def test_review_details_extracted(): + body = ( + "**Actionable comments posted: 0**\n\n" + "
\n\U0001f4dc Review details\n\n" + "**Configuration used**: CHILL\n\n" + "
" + ) + sections = parse_status_comment(body) + assert "review_details" in sections + + +def test_additional_comments_extracted(): + body = ( + "**Actionable comments posted: 0**\n\n" + "
\n\U0001f509 Additional comments (3)
\n\n" + "LGTM! Good stuff.\n\n" + "
" + ) + sections = parse_status_comment(body) + assert "additional_comments" in sections + + +def test_additional_context_extracted(): + body = ( + "**Actionable comments posted: 0**\n\n" + "
\n\U0001f9f0 Additional context used\n\n" + "Code graph analysis...\n\n" + "
" + ) + sections = parse_status_comment(body) + assert "additional_context" in sections + + +def test_multiple_sections_in_one_status(): + body = ( + "**Actionable comments posted: 2**\n\n" + "
\n\U0001f916 Fix all issues with AI agents\n\nFix A\n\n
\n\n" + "
\n\U0001f9f9 Nitpick comments (1)
\n\n" + "
\na.py (1)
\n\n" + "`1-5`: **Nit**\n\nDesc.\n\n" + "
\n\n" + "
\n\n" + "
\n\U0001f4dc Review details\nConfig\n
\n\n" + "
\n\U0001f509 Additional comments (1)
\nLGTM\n
" + ) + sections = parse_status_comment(body) + assert "actionable_summary" in sections + assert "nitpick" in sections + assert "review_details" in sections + assert "additional_comments" in sections + + +def test_outside_diff_strips_blockquote_prefixes(): + """Outside-diff content uses markdown blockquote > prefixes that must be stripped.""" + body = ( + "**Actionable comments posted: 1**\n\n" + "> [!CAUTION]\n" + "> Some comments are outside the diff\n>\n" + "
\n\u26a0\ufe0f Outside diff range comments (1)
\n\n" + "
\nsrc/init.py (1)
\n\n" + "> \n" + "> `59-70`: **Hidden attributes issue.**\n" + "> \n" + "> The filter is only applied inside the search branch.\n" + "> \n" + "
\n\n" + "
" + ) + sections = parse_status_comment(body) + assert "outside_diff" in sections + assert len(sections["outside_diff"]) == 1 + item = sections["outside_diff"][0] + assert item["path"] == "src/init.py" + assert item["line"] == 59 + # Body should not have > prefixes + assert not item["body"].startswith(">") + assert "Hidden attributes issue." in item["body"] + + +def test_nitpick_line_range_extracted(): + body = ( + "**Actionable comments posted: 0**\n\n" + "
\n\U0001f9f9 Nitpick comments (1)
\n\n" + "
\nsrc/util.py (1)
\n\n" + "`304-326`: **Add CLIENTS branch**\n\nSome description.\n\n" + "
\n\n" + "
" + ) + sections = parse_status_comment(body) + assert sections["nitpick"][0]["line"] == 304 + assert "Add CLIENTS branch" in sections["nitpick"][0]["body"] diff --git a/offline/tests/test_details_splitter.py b/offline/tests/test_details_splitter.py new file mode 100644 index 0000000..74179c0 --- /dev/null +++ b/offline/tests/test_details_splitter.py @@ -0,0 +1,69 @@ +"""Tests for DetailsBlockSplitter.""" + +from __future__ import annotations + +from code_review_benchmark.parsers.coderabbit import split_details_blocks + + +def test_single_details_block(): + html = "
Title\nContent\n
" + blocks = split_details_blocks(html) + assert len(blocks) == 1 + assert "Title" in blocks[0] + assert "Content" in blocks[0] + + +def test_multiple_top_level_blocks(): + html = "
First\nA\n
\n" "
Second\nB\n
" + blocks = split_details_blocks(html) + assert len(blocks) == 2 + assert "First" in blocks[0] + assert "Second" in blocks[1] + + +def test_nested_details_counted_as_one(): + html = ( + "
Outer\n" + "
Inner\nNested\n
\n" + "
" + ) + blocks = split_details_blocks(html) + assert len(blocks) == 1 + assert "Outer" in blocks[0] + assert "Inner" in blocks[0] + + +def test_deeply_nested(): + html = ( + "
L1\n" + "
L2\n" + "
L3\nDeep\n
\n" + "
\n" + "
" + ) + blocks = split_details_blocks(html) + assert len(blocks) == 1 + assert "Deep" in blocks[0] + + +def test_text_outside_details_ignored(): + html = "Some preamble\n
A\nX\n
\nTrailing text" + blocks = split_details_blocks(html) + assert len(blocks) == 1 + assert "preamble" not in blocks[0] + assert "Trailing" not in blocks[0] + + +def test_empty_input(): + assert split_details_blocks("") == [] + + +def test_no_details_tags(): + assert split_details_blocks("Just plain text") == [] + + +def test_extract_summary(): + html = "
\U0001f9f9 Nitpick comments (4)\nStuff\n
" + blocks = split_details_blocks(html) + assert len(blocks) == 1 + assert "\U0001f9f9 Nitpick comments (4)" in blocks[0] diff --git a/offline/tests/test_integration_coderabbit.py b/offline/tests/test_integration_coderabbit.py new file mode 100644 index 0000000..c3b314b --- /dev/null +++ b/offline/tests/test_integration_coderabbit.py @@ -0,0 +1,90 @@ +"""Integration tests: run CodeRabbitParser on real benchmark data.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from code_review_benchmark.parsers.coderabbit import CodeRabbitParser + +BENCHMARK_FILE = Path(__file__).resolve().parents[1] / "results" / "benchmark_data.json" + + +@pytest.mark.skipif(not BENCHMARK_FILE.exists(), reason="benchmark_data.json not available") +def test_parse_all_coderabbit_reviews(): + """Parse every CodeRabbit review in the benchmark and verify no crashes.""" + with open(BENCHMARK_FILE) as f: + data = json.load(f) + + parser = CodeRabbitParser() + total_reviews = 0 + total_inline = 0 + total_nitpick = 0 + total_actionable = 0 + total_outside = 0 + + for golden_url, entry in data.items(): + for review in entry.get("reviews", []): + if review["tool"] != "coderabbit": + continue + total_reviews += 1 + comments = review.get("review_comments", []) + result = parser.parse(comments) + + # Every review should produce some sections + assert isinstance(result.sections, dict) + + for section_name, parsed_comments in result.sections.items(): + for pc in parsed_comments: + # Every parsed comment should have a non-empty body + assert pc.body, f"Empty body in {section_name} for {golden_url}" + assert pc.section == section_name + + total_inline += len(result.sections.get("inline", [])) + total_nitpick += len(result.sections.get("nitpick", [])) + total_actionable += len(result.sections.get("actionable_summary", [])) + total_outside += len(result.sections.get("outside_diff", [])) + + print(f"\nParsed {total_reviews} CodeRabbit reviews:") + print(f" Inline comments: {total_inline}") + print(f" Nitpick comments: {total_nitpick}") + print(f" Actionable summaries: {total_actionable}") + print(f" Outside-diff comments: {total_outside}") + + assert total_reviews > 0, "No CodeRabbit reviews found" + assert total_inline > 0, "Expected some inline comments" + + +@pytest.mark.skipif(not BENCHMARK_FILE.exists(), reason="benchmark_data.json not available") +def test_to_markdown_produces_clean_output(): + """Verify rendered markdown doesn't contain noise patterns.""" + with open(BENCHMARK_FILE) as f: + data = json.load(f) + + parser = CodeRabbitParser() + noise_patterns = [ + "\nWalkthrough text", + "path": None, + "line": None, + "created_at": "2026-01-26T00:01:00Z", + }, + ] + result = step1_5.build_review_output("coderabbit", comments, sections_config) + assert result["tool"] == "coderabbit" + # Inline is included + assert len(result["review_comments"]) == 1 + assert result["review_comments"][0]["section"] == "inline" + # Walkthrough is excluded + assert len(result["excluded_comments"]) == 1 + assert result["excluded_comments"][0]["section"] == "walkthrough" + # Rendered markdown contains only included comments + assert "Null check missing" in result["rendered_markdown"] + assert "Walkthrough" not in result["rendered_markdown"] + + +def test_run_parser_end_to_end(tmp_path, monkeypatch): + """Integration test: parse benchmark data and produce output file.""" + data = { + "https://example/pr1": { + "reviews": [ + { + "tool": "coderabbit", + "review_comments": [ + { + "body": "_\u26a0\ufe0f Potential issue_ | _\U0001f7e0 Major_\n\nBug found.", + "path": "src/main.py", + "line": 5, + "created_at": "2026-01-26T00:00:00Z", + }, + ], + }, + { + "tool": "claude", + "review_comments": [ + { + "body": "Some claude comment", + "path": None, + "line": None, + "created_at": "2026-01-26T00:00:00Z", + }, + ], + }, + ], + }, + } + results_dir = tmp_path + benchmark_file = results_dir / "benchmark_data.json" + benchmark_file.write_text(json.dumps(data)) + + monkeypatch.setattr(step1_5, "RESULTS_DIR", results_dir) + monkeypatch.setattr(step1_5, "BENCHMARK_DATA_FILE", benchmark_file) + + sections_config = step1_5.resolve_sections(include=None, exclude=None, only=None) + step1_5.run_parser("coderabbit", sections_config) + + output_file = results_dir / "parsed_coderabbit.json" + assert output_file.exists() + + output = json.loads(output_file.read_text()) + assert output["config"]["tool"] == "coderabbit" + assert "inline" in output["config"]["included_sections"] + assert "https://example/pr1" in output["reviews"] + review = output["reviews"]["https://example/pr1"] + assert review["tool"] == "coderabbit" + assert len(review["review_comments"]) >= 1 + assert review["review_comments"][0]["section"] == "inline" + assert "Bug found" in review["rendered_markdown"] diff --git a/offline/tests/test_step2.py b/offline/tests/test_step2.py index 86f40f3..c628e33 100644 --- a/offline/tests/test_step2.py +++ b/offline/tests/test_step2.py @@ -121,3 +121,62 @@ class SimpleNamespace: def __init__(self, **kwargs): self.__dict__.update(kwargs) + + +def test_get_comment_text_for_review_uses_parsed_file(tmp_path, monkeypatch): + """When a parsed file exists, use rendered_markdown from it.""" + monkeypatch.setattr(step2, "RESULTS_DIR", tmp_path) + + parsed = { + "config": {"tool": "coderabbit"}, + "reviews": { + "https://example/pr": { + "tool": "coderabbit", + "review_comments": [], + "excluded_comments": [], + "rendered_markdown": "## Inline comment — a.py:1 [critical]\n\nClean issue text", + } + }, + } + parsed_file = tmp_path / "parsed_coderabbit.json" + parsed_file.write_text(json.dumps(parsed)) + + result = step2.get_comment_text_for_review( + "https://example/pr", + "coderabbit", + [{"body": "Raw noisy comment", "path": None, "line": None}], + ) + assert result == "## Inline comment — a.py:1 [critical]\n\nClean issue text" + + +def test_get_comment_text_for_review_fallback(tmp_path, monkeypatch): + """When no parsed file exists, fall back to raw comments.""" + monkeypatch.setattr(step2, "RESULTS_DIR", tmp_path) + + result = step2.get_comment_text_for_review( + "https://example/pr", + "unknown-tool", + [{"body": "Raw comment A"}, {"body": "Raw comment B"}], + ) + assert "Raw comment A" in result + assert "Raw comment B" in result + assert "---" in result + + +def test_get_comment_text_for_review_missing_pr(tmp_path, monkeypatch): + """When parsed file exists but PR not in it, fall back to raw comments.""" + monkeypatch.setattr(step2, "RESULTS_DIR", tmp_path) + + parsed = { + "config": {"tool": "coderabbit"}, + "reviews": {}, + } + parsed_file = tmp_path / "parsed_coderabbit.json" + parsed_file.write_text(json.dumps(parsed)) + + result = step2.get_comment_text_for_review( + "https://example/missing-pr", + "coderabbit", + [{"body": "Fallback text"}], + ) + assert "Fallback text" in result From 0d2998ea8f47e213e66a6eae87d4281a03086750 Mon Sep 17 00:00:00 2001 From: taisazero Date: Mon, 13 Apr 2026 15:11:29 -0700 Subject: [PATCH 2/6] Refactor section handling in review parsing - Removed hardcoded DEFAULT_SECTIONS and replaced it with a method in parsers to define default sections. - Updated `resolve_sections` function to accept dynamic defaults from the parser. - Added `default_sections` method to BaseParser and implemented it in CodeRabbitParser and DefaultParser. - Adjusted tests to validate the new default section handling. --- offline/code_review_benchmark/parsers/base.py | 9 ++++ .../parsers/coderabbit.py | 15 +++++++ .../code_review_benchmark/parsers/default.py | 3 ++ .../step1_5_parse_reviews.py | 31 ++++++------- offline/tests/test_step1_5.py | 43 ++++++++++++++----- 5 files changed, 72 insertions(+), 29 deletions(-) diff --git a/offline/code_review_benchmark/parsers/base.py b/offline/code_review_benchmark/parsers/base.py index 6b6ce09..050fb6d 100644 --- a/offline/code_review_benchmark/parsers/base.py +++ b/offline/code_review_benchmark/parsers/base.py @@ -66,3 +66,12 @@ class BaseParser(ABC): def parse(self, review_comments: list[dict]) -> ParsedReview: """Parse raw review comments into categorized sections.""" ... + + @abstractmethod + def default_sections(self) -> dict[str, bool]: + """Section names this parser produces and their default include/exclude. + + Keys are all section names the parser can emit. + Values are True (included by default) or False (excluded by default). + """ + ... diff --git a/offline/code_review_benchmark/parsers/coderabbit.py b/offline/code_review_benchmark/parsers/coderabbit.py index 50b269c..ede8b2f 100644 --- a/offline/code_review_benchmark/parsers/coderabbit.py +++ b/offline/code_review_benchmark/parsers/coderabbit.py @@ -293,6 +293,21 @@ def parse_status_comment(body: str) -> dict[str, list[dict]]: class CodeRabbitParser(BaseParser): """Parses CodeRabbit structured comments into categorized sections.""" + def default_sections(self) -> dict[str, bool]: + return { + "inline": True, + "actionable_summary": True, + "outside_diff": True, + "nitpick": False, + "walkthrough": False, + "pre_merge_checks": False, + "finishing_touches": False, + "review_details": False, + "additional_comments": False, + "additional_context": False, + "unknown": False, + } + def parse(self, review_comments: list[dict]) -> ParsedReview: sections: dict[str, list[ParsedComment]] = {} diff --git a/offline/code_review_benchmark/parsers/default.py b/offline/code_review_benchmark/parsers/default.py index e3b3d07..08dd022 100644 --- a/offline/code_review_benchmark/parsers/default.py +++ b/offline/code_review_benchmark/parsers/default.py @@ -10,6 +10,9 @@ class DefaultParser(BaseParser): """Passes all comments through as a single 'raw' section.""" + def default_sections(self) -> dict[str, bool]: + return {"raw": True} + def parse(self, review_comments: list[dict]) -> ParsedReview: comments = [ ParsedComment( diff --git a/offline/code_review_benchmark/step1_5_parse_reviews.py b/offline/code_review_benchmark/step1_5_parse_reviews.py index 1aee378..75ba0e2 100644 --- a/offline/code_review_benchmark/step1_5_parse_reviews.py +++ b/offline/code_review_benchmark/step1_5_parse_reviews.py @@ -17,30 +17,23 @@ RESULTS_DIR = Path("results") BENCHMARK_DATA_FILE = RESULTS_DIR / "benchmark_data.json" -DEFAULT_SECTIONS = { - "inline": True, - "actionable_summary": True, - "outside_diff": True, - "nitpick": False, - "walkthrough": False, - "pre_merge_checks": False, - "finishing_touches": False, - "review_details": False, - "additional_comments": False, - "additional_context": False, - "unknown": False, -} - def resolve_sections( include: list[str] | None, exclude: list[str] | None, only: list[str] | None, + defaults: dict[str, bool] | None = None, ) -> dict[str, bool]: - """Resolve section filter config from CLI flags.""" + """Resolve section filter config from CLI flags. + + *defaults* comes from the parser's ``default_sections()``; when ``None`` + the caller must supply it via get_parser(tool).default_sections(). + """ + if defaults is None: + defaults = {} if only: - return {name: (name in only) for name in DEFAULT_SECTIONS} - config = dict(DEFAULT_SECTIONS) + return {name: (name in only) for name in defaults} + config = dict(defaults) if include: for name in include: if name in config: @@ -239,7 +232,9 @@ def main(): args = parser.parse_args() only = args.only.split(",") if args.only else None - sections_config = resolve_sections(args.include, args.exclude, only) + tool_parser = get_parser(args.tool) + defaults = tool_parser.default_sections() + sections_config = resolve_sections(args.include, args.exclude, only, defaults) if args.preview: if not BENCHMARK_DATA_FILE.exists(): diff --git a/offline/tests/test_step1_5.py b/offline/tests/test_step1_5.py index 9fcbdc3..ec1cda8 100644 --- a/offline/tests/test_step1_5.py +++ b/offline/tests/test_step1_5.py @@ -5,36 +5,56 @@ import json from code_review_benchmark import step1_5_parse_reviews as step1_5 +from code_review_benchmark.parsers import get_parser -def test_default_sections(): - assert step1_5.DEFAULT_SECTIONS["inline"] is True - assert step1_5.DEFAULT_SECTIONS["actionable_summary"] is True - assert step1_5.DEFAULT_SECTIONS["outside_diff"] is True - assert step1_5.DEFAULT_SECTIONS["nitpick"] is False - assert step1_5.DEFAULT_SECTIONS["walkthrough"] is False +# Helpers ---------------------------------------------------------------- + +def _coderabbit_defaults() -> dict[str, bool]: + return get_parser("coderabbit").default_sections() + + +# Tests ------------------------------------------------------------------ + +def test_parser_declares_default_sections(): + """Each parser must declare its own default sections.""" + defaults = _coderabbit_defaults() + assert defaults["inline"] is True + assert defaults["actionable_summary"] is True + assert defaults["outside_diff"] is True + assert defaults["nitpick"] is False + assert defaults["walkthrough"] is False + + +def test_default_parser_declares_sections(): + defaults = get_parser("unknown_tool").default_sections() + assert defaults == {"raw": True} def test_resolve_sections_defaults(): - result = step1_5.resolve_sections(include=None, exclude=None, only=None) + defaults = _coderabbit_defaults() + result = step1_5.resolve_sections(include=None, exclude=None, only=None, defaults=defaults) assert result["inline"] is True assert result["nitpick"] is False def test_resolve_sections_include(): - result = step1_5.resolve_sections(include=["nitpick"], exclude=None, only=None) + defaults = _coderabbit_defaults() + result = step1_5.resolve_sections(include=["nitpick"], exclude=None, only=None, defaults=defaults) assert result["nitpick"] is True assert result["inline"] is True # default still on def test_resolve_sections_exclude(): - result = step1_5.resolve_sections(include=None, exclude=["outside_diff"], only=None) + defaults = _coderabbit_defaults() + result = step1_5.resolve_sections(include=None, exclude=["outside_diff"], only=None, defaults=defaults) assert result["outside_diff"] is False assert result["inline"] is True def test_resolve_sections_only(): - result = step1_5.resolve_sections(include=None, exclude=None, only=["inline"]) + defaults = _coderabbit_defaults() + result = step1_5.resolve_sections(include=None, exclude=None, only=["inline"], defaults=defaults) assert result["inline"] is True assert result["actionable_summary"] is False assert result["outside_diff"] is False @@ -119,7 +139,8 @@ def test_run_parser_end_to_end(tmp_path, monkeypatch): monkeypatch.setattr(step1_5, "RESULTS_DIR", results_dir) monkeypatch.setattr(step1_5, "BENCHMARK_DATA_FILE", benchmark_file) - sections_config = step1_5.resolve_sections(include=None, exclude=None, only=None) + defaults = get_parser("coderabbit").default_sections() + sections_config = step1_5.resolve_sections(include=None, exclude=None, only=None, defaults=defaults) step1_5.run_parser("coderabbit", sections_config) output_file = results_dir / "parsed_coderabbit.json" From 61ead146420938ca62e6353f598de0d0b9d940ea Mon Sep 17 00:00:00 2001 From: taisazero Date: Mon, 13 Apr 2026 15:28:57 -0700 Subject: [PATCH 3/6] exclude actionable summary --- offline/code_review_benchmark/parsers/coderabbit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offline/code_review_benchmark/parsers/coderabbit.py b/offline/code_review_benchmark/parsers/coderabbit.py index ede8b2f..a4ae488 100644 --- a/offline/code_review_benchmark/parsers/coderabbit.py +++ b/offline/code_review_benchmark/parsers/coderabbit.py @@ -296,7 +296,7 @@ class CodeRabbitParser(BaseParser): def default_sections(self) -> dict[str, bool]: return { "inline": True, - "actionable_summary": True, + "actionable_summary": False, "outside_diff": True, "nitpick": False, "walkthrough": False, From f327a90f41f4de498c06e796dbf64d88d5b5b1f5 Mon Sep 17 00:00:00 2001 From: taisazero Date: Mon, 13 Apr 2026 18:27:47 -0700 Subject: [PATCH 4/6] Add function to retrieve comment text for reviews - Introduced `get_comment_text_for_review` to fetch comment text from a parsed file if available, falling back to the original comment extraction method otherwise. - Updated the `main` function to utilize the new comment retrieval method for improved handling of review comments. --- .../step2_extract_comments.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/offline/code_review_benchmark/step2_extract_comments.py b/offline/code_review_benchmark/step2_extract_comments.py index 3d3ec9b..6dad788 100644 --- a/offline/code_review_benchmark/step2_extract_comments.py +++ b/offline/code_review_benchmark/step2_extract_comments.py @@ -149,6 +149,23 @@ def get_all_comment_text(review_comments: list[dict]) -> str: return "\n\n---\n\n".join(bodies) +def get_comment_text_for_review(golden_url: str, tool: str, review_comments: list[dict]) -> str: + """Get comment text — use parsed file if available, otherwise raw comments.""" + parsed_file = RESULTS_DIR / f"parsed_{tool}.json" + + if parsed_file.exists(): + with open(parsed_file) as f: + parsed_data = json.load(f) + review = parsed_data.get("reviews", {}).get(golden_url) + if review is not None: + # Parsed file has an entry for this PR — use it even if empty + # (empty means the parser found no actionable comments) + return review.get("rendered_markdown", "") + + # Fallback: original behavior (no parsed file for this tool) + return get_all_comment_text(review_comments) + + async def process_batch(tasks: list, batch_size: int = BATCH_SIZE) -> list: """Process async tasks in batches.""" results = [] @@ -217,7 +234,7 @@ async def main(): continue comments = review.get("review_comments", []) - all_text = get_all_comment_text(comments) + all_text = get_comment_text_for_review(golden_url, tool, comments) if all_text and len(all_text.strip()) >= 20: extraction_tasks.append((golden_url, tool, all_text)) From df26a5ecbaa70b59984c2ad2a104561108bb40b8 Mon Sep 17 00:00:00 2001 From: taisazero Date: Mon, 13 Apr 2026 18:42:34 -0700 Subject: [PATCH 5/6] Remove unused details block regex and update actionable summary default in tests --- offline/code_review_benchmark/parsers/coderabbit.py | 4 ---- offline/tests/test_step1_5.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/offline/code_review_benchmark/parsers/coderabbit.py b/offline/code_review_benchmark/parsers/coderabbit.py index a4ae488..0eb2161 100644 --- a/offline/code_review_benchmark/parsers/coderabbit.py +++ b/offline/code_review_benchmark/parsers/coderabbit.py @@ -33,10 +33,6 @@ # --------------------------------------------------------------------------- _HTML_COMMENT = re.compile(r"", re.DOTALL) -_DETAILS_BLOCK_RE = re.compile( - r"
\s*(.*?).*?
", - re.DOTALL, -) _NOISE_SUMMARIES = { "\U0001f9e9 Analysis chain", "\U0001f916 Prompt for AI Agents", diff --git a/offline/tests/test_step1_5.py b/offline/tests/test_step1_5.py index ec1cda8..7da5f9d 100644 --- a/offline/tests/test_step1_5.py +++ b/offline/tests/test_step1_5.py @@ -20,7 +20,7 @@ def test_parser_declares_default_sections(): """Each parser must declare its own default sections.""" defaults = _coderabbit_defaults() assert defaults["inline"] is True - assert defaults["actionable_summary"] is True + assert defaults["actionable_summary"] is False assert defaults["outside_diff"] is True assert defaults["nitpick"] is False assert defaults["walkthrough"] is False From 380840d29631738efa14d19bf72b49d2652e1a78 Mon Sep 17 00:00:00 2001 From: taisazero Date: Wed, 5 Aug 2026 07:38:20 -0700 Subject: [PATCH 6/6] Add parsers README: framework guide and extension walkthrough --- offline/README.md | 16 ++ .../code_review_benchmark/parsers/README.md | 226 ++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 offline/code_review_benchmark/parsers/README.md diff --git a/offline/README.md b/offline/README.md index 3aa9456..db91886 100644 --- a/offline/README.md +++ b/offline/README.md @@ -109,6 +109,22 @@ uv run python -m code_review_benchmark.step1_download_prs --output results/bench **Output:** `results/benchmark_data.json` +### 1.5. Parse reviews + +Normalize one tool's raw review comments into clean, named sections before LLM extraction. Registered tools use their tool-specific parser; other tools use a passthrough parser. + +```bash +# Parse reviews with the tool's default section filters +uv run python -m code_review_benchmark.step1_5_parse_reviews --tool coderabbit + +# Preview the rendered markdown without writing the output file +uv run python -m code_review_benchmark.step1_5_parse_reviews --tool coderabbit --preview +``` + +**Output:** `results/parsed_{tool}.json` + +See the [parser framework guide](code_review_benchmark/parsers/README.md) for parser behavior, CLI options, and an extension walkthrough. + ### 2. Extract comments Extract individual issues from review comments for matching: diff --git a/offline/code_review_benchmark/parsers/README.md b/offline/code_review_benchmark/parsers/README.md new file mode 100644 index 0000000..7acd7fe --- /dev/null +++ b/offline/code_review_benchmark/parsers/README.md @@ -0,0 +1,226 @@ +# Review comment parsers + +Raw tool review comments mix useful findings with tool-specific noise such as analysis chains, AI-agent prompts, linter output, severity badges, and HTML comments. They can also embed structure, including status or summary comments with nested `
` sections. Parsers strip that noise and classify comments into named sections before LLM extraction so the benchmark judges clean, comparable input. Each tool can provide its own parser; tools without one use a passthrough parser. + +## Where it sits in the pipeline + +```text +Step 1 Step 1.5 Step 2 Step 2.5 Step 3 +benchmark_data.json --> parsed_{tool}.json --> extract comments --> deduplicate --> judge + | ^ + +---- raw comments when parsed file/entry ----+ + is absent (auto-detected by step 2) +``` + +Step 1.5 is optional. When `results/parsed_{tool}.json` is absent, step 2 reads the raw `review_comments` from `results/benchmark_data.json`. + +## Components + +| File | Role | +|---|---| +| [`base.py`](base.py) | Defines `BaseParser`, `ParsedComment`, `ParsedReview`, and markdown rendering. | +| [`__init__.py`](__init__.py) | Owns the parser registry and the `get_parser(tool)` lookup. | +| [`default.py`](default.py) | Provides the `DefaultParser` passthrough for tools without a registered parser. | +| [`coderabbit.py`](coderabbit.py) | Implements CodeRabbit-specific classification, structured-section parsing, and noise removal. | +| [`../step1_5_parse_reviews.py`](../step1_5_parse_reviews.py) | Exposes filtering and preview controls and writes `results/parsed_{tool}.json`. | + +## Core contracts + +### Parser input and output + +Every parser subclasses `BaseParser` and implements `BaseParser.parse(review_comments: list[dict]) -> ParsedReview`: + +```python +def parse(self, review_comments: list[dict]) -> ParsedReview: + ... +``` + +Each input dictionary has the shape used by `review_comments` entries in `benchmark_data.json`: `body`, `path`, `line`, and `created_at`. + +A `ParsedComment` preserves those four values and adds two parser-owned fields: + +| Field | Type | Meaning | +|---|---|---| +| `body` | `str` | Cleaned comment text. | +| `path` | `str \| None` | Referenced file, if any. | +| `line` | `int \| None` | Referenced line, if any. | +| `created_at` | `str` | Original creation timestamp. | +| `section` | `str` | Named category emitted by the parser. | +| `severity` | `str \| None` | Optional severity metadata. | + +`ParsedReview.sections` is a dictionary from section name to a list of `ParsedComment` objects. + +### Section defaults and filtering + +Each parser also implements: + +```python +def default_sections(self) -> dict[str, bool]: + ... +``` + +Every section name that `parse()` can emit must be a key in this dictionary. `True` includes the section by default; `False` excludes it by default while still allowing a caller to opt in. + +This declaration is required, not advisory. A section emitted by `parse()` but missing from `default_sections()` is silently excluded and cannot be enabled from the CLI: `resolve_sections()` only toggles declared keys, and `build_review_output()` evaluates undeclared sections with `sections_config.get(name, False)`. + +Excluded comments are not deleted. Step 1.5 writes them to `excluded_comments` in `parsed_{tool}.json` for auditability. + +`severity` is optional metadata. The CLI accepts `critical`, `major`, and `minor` for `--min-severity`; those are the only recognized ranks. Comments with `severity=None` are never severity-filtered. Custom parsers should use only the recognized values: any other nonempty value has rank 0 and is excluded when a minimum severity is active. + +### Registration and fallback + +The `PARSERS` dictionary in `parsers/__init__.py` is populated lazily by `_register_parsers()`. `get_parser(tool)` returns an instance of the registered parser or falls back to `DefaultParser`, which emits every input comment in an included `raw` section. + +### Rendering and step 2 integration + +`ParsedReview.to_markdown()` renders each comment as a section header followed by its body, with comments separated by `---`. A comment with all metadata uses this form: + +```markdown +## {Section Label} — {path}:{line} [{severity}] + +{body} +``` + +The path and line suffix is omitted when there is no complete location, and the severity suffix is omitted when `severity` is `None`. Section labels replace underscores with spaces and use title case. + +This `rendered_markdown` is the text consumed by step 2's LLM extraction. If `results/parsed_{tool}.json` exists and its `reviews` map contains the PR, step 2 uses that entry's `rendered_markdown` even when it is empty. Empty markdown means the parser found nothing actionable, so the review is effectively skipped by step 2's minimum 20-character check. If the parsed file or PR entry is absent, `get_comment_text_for_review()` in [`../step2_extract_comments.py`](../step2_extract_comments.py) falls back to joining the raw comment bodies. + +## Adding a parser for your tool + +The following example adds an `acme` parser that recognizes inline findings and summary comments, extracts supported severity metadata, removes Acme's hidden analysis blocks, and excludes unclassified comments by default. + +1. Create `code_review_benchmark/parsers/acme.py`: + +```python +from __future__ import annotations + +import re + +from code_review_benchmark.parsers.base import BaseParser +from code_review_benchmark.parsers.base import ParsedComment +from code_review_benchmark.parsers.base import ParsedReview + +_SEVERITY_RE = re.compile(r"^\s*", re.IGNORECASE) +_ANALYSIS_RE = re.compile( + r"
\s*Acme analysis.*?
", + re.IGNORECASE | re.DOTALL, +) +_SUMMARY_RE = re.compile(r"^## Acme review summary\s*", re.IGNORECASE) + + +class AcmeParser(BaseParser): + def default_sections(self) -> dict[str, bool]: + return {"inline": True, "summary": True, "other": False} + + def parse(self, review_comments: list[dict]) -> ParsedReview: + sections: dict[str, list[ParsedComment]] = {} + for comment in review_comments: + body = comment.get("body", "") + severity_match = _SEVERITY_RE.match(body) + severity = severity_match.group(1).lower() if severity_match else None + body = _SEVERITY_RE.sub("", body, count=1) + body = _ANALYSIS_RE.sub("", body).strip() + + if comment.get("path") and comment.get("line") is not None: + section = "inline" + elif _SUMMARY_RE.match(body): + section = "summary" + body = _SUMMARY_RE.sub("", body, count=1).strip() + else: + section = "other" + + parsed = ParsedComment( + body=body, + path=comment.get("path"), + line=comment.get("line"), + created_at=comment.get("created_at", ""), + section=section, + severity=severity, + ) + sections.setdefault(section, []).append(parsed) + return ParsedReview(sections=sections) +``` + +2. Import and register it in `_register_parsers()` in `parsers/__init__.py`: + +```python +def _register_parsers() -> None: + global _registered + if _registered: + return + _registered = True + try: + from code_review_benchmark.parsers.acme import AcmeParser + from code_review_benchmark.parsers.coderabbit import CodeRabbitParser + + PARSERS["acme"] = AcmeParser + PARSERS["coderabbit"] = CodeRabbitParser + except ImportError: + pass +``` + +3. Parse Acme reviews from `results/benchmark_data.json`: + +```bash +uv run python -m code_review_benchmark.step1_5_parse_reviews --tool acme +``` + +4. Inspect the rendered input before writing the output file: + +```bash +uv run python -m code_review_benchmark.step1_5_parse_reviews --tool acme --preview +``` + +5. Add tests that mirror the focused parser coverage in `offline/tests/test_parsers_*.py` and the classification, cleanup, structured parsing, and integration coverage in `offline/tests/test_coderabbit_*.py`. + +## The CodeRabbit parser as a reference implementation + +`CodeRabbitParser` demonstrates a parser for a tool with several comment formats. Its classifier applies a deliberate priority: a comment with both `path` and `line` is inline, then known HTML markers identify structured comment types, and finally content markers identify status and standalone outside-diff comments. + +For structured status comments, `split_details_blocks()` tracks nesting depth so top-level `
` blocks can contain nested file groups without being split incorrectly. Inline cleanup removes blocks whose summaries match `_NOISE_SUMMARIES`, then removes HTML comments and severity-badge lines. `_SEVERITY_PATTERNS` extracts `critical`, `major`, or `minor`; `_SECTION_SUMMARY_MAP` maps status-comment summaries to benchmark section names. The `nitpick` and `outside_diff` sections receive additional file-grouped parsing that recovers file paths and starting line numbers from their nested blocks. + +## CLI reference + +Run the CLI from `offline/` with `uv run python -m code_review_benchmark.step1_5_parse_reviews`. + +| Flag | Behavior | +|---|---| +| `--tool TOOL` | Parser and review tool to process. Defaults to `coderabbit`; unknown names use `DefaultParser`. | +| `--include [SECTION ...]` | Include additional declared sections on top of the parser defaults. | +| `--exclude [SECTION ...]` | Exclude declared sections from the parser defaults. | +| `--only SECTION[,SECTION...]` | Include only the comma-separated declared sections. | +| `--preview` | Print each matching PR's rendered markdown without writing `parsed_{tool}.json`. | +| `--min-severity {minor,major,critical}` | Exclude comments with recognized severity below the selected rank. Comments without severity remain included. | +| `--write-candidates` | Write included parsed comments directly to `results/{model}/candidates.json`, bypassing step 2. | +| `--model-dir DIR` | Select the results subdirectory used with `--write-candidates`. | +| `--include-path` | Prefix candidate text with `path` or `path:line` when used with `--write-candidates`. | + +Without `--preview` or `--write-candidates`, step 1.5 writes this shape to `results/parsed_{tool}.json`: + +```json +{ + "config": { + "tool": "acme", + "included_sections": ["inline", "summary"], + "excluded_sections": ["other"], + "min_severity": null + }, + "reviews": { + "https://github.com/example/project/pull/123": { + "tool": "acme", + "review_comments": [ + { + "body": "This can dereference None.", + "path": "src/service.py", + "line": 42, + "created_at": "2026-01-01T00:00:00Z", + "section": "inline", + "severity": "major" + } + ], + "excluded_comments": [], + "rendered_markdown": "## Inline — src/service.py:42 [major]\n\nThis can dereference None." + } + } +} +```