diff --git a/src/openamp_foundry/evidence/length_distribution_report.py b/src/openamp_foundry/evidence/length_distribution_report.py new file mode 100644 index 00000000..e1e7bb87 --- /dev/null +++ b/src/openamp_foundry/evidence/length_distribution_report.py @@ -0,0 +1,190 @@ +"""Length distribution report — Phase C C5. + +Detects whether benchmark improvements could be explained by a length shortcut: +the model predicts positives because AMPs tend to be 10–40 residues long, +not because of genuine activity signal. + +Raises a 'length_shortcut_likely' flag when the fraction of positives in the +typical AMP length range significantly exceeds the fraction of negatives in the +same range. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +AMP_LENGTH_MIN = 10 +AMP_LENGTH_MAX = 40 +SHORTCUT_RATIO_THRESHOLD = 1.5 +SHORTCUT_FRACTION_THRESHOLD = 0.60 + + +@dataclass +class LengthDistributionReport: + n_positives: int + n_negatives: int + positive_lengths: list[int] = field(default_factory=list) + negative_lengths: list[int] = field(default_factory=list) + positive_mean_length: float = 0.0 + positive_median_length: float = 0.0 + negative_mean_length: float = 0.0 + negative_median_length: float = 0.0 + positive_in_amp_range_fraction: float = 0.0 + negative_in_amp_range_fraction: float = 0.0 + length_ratio: float = 0.0 + length_shortcut_likely: bool = False + shortcut_explanation: str = "" + amp_length_min: int = AMP_LENGTH_MIN + amp_length_max: int = AMP_LENGTH_MAX + ratio_threshold: float = SHORTCUT_RATIO_THRESHOLD + fraction_threshold: float = SHORTCUT_FRACTION_THRESHOLD + + +def _mean(values: list[float]) -> float: + if not values: + return 0.0 + return sum(values) / len(values) + + +def _median(values: list[float]) -> float: + if not values: + return 0.0 + s = sorted(values) + n = len(s) + mid = n // 2 + if n % 2 == 0: + return (s[mid - 1] + s[mid]) / 2.0 + return s[mid] + + +def _in_range_fraction(lengths: list[int], lo: int, hi: int) -> float: + if not lengths: + return 0.0 + return sum(1 for ln in lengths if lo <= ln <= hi) / len(lengths) + + +def compute_length_report( + sequences: list[str], + labels: list[int], + amp_length_min: int = AMP_LENGTH_MIN, + amp_length_max: int = AMP_LENGTH_MAX, + ratio_threshold: float = SHORTCUT_RATIO_THRESHOLD, + fraction_threshold: float = SHORTCUT_FRACTION_THRESHOLD, +) -> LengthDistributionReport: + """Compute length-distribution shortcut report for a benchmark dataset. + + Args: + sequences: Amino acid sequences in the benchmark. + labels: Binary labels (1=positive/AMP, 0=negative/non-AMP). + amp_length_min: Lower bound of typical AMP length range. + amp_length_max: Upper bound of typical AMP length range. + ratio_threshold: Ratio of pos/neg in-range fraction above which shortcut is flagged. + fraction_threshold: Fraction of positives in range above which shortcut is flagged + (combined with ratio check). + + Returns: + LengthDistributionReport with per-group statistics and shortcut flag. + + Raises: + ValueError: If sequences and labels have different lengths. + """ + if len(sequences) != len(labels): + raise ValueError( + f"sequences and labels must have the same length, " + f"got {len(sequences)} and {len(labels)}" + ) + + pos_lengths = [len(seq) for seq, lbl in zip(sequences, labels) if lbl == 1] + neg_lengths = [len(seq) for seq, lbl in zip(sequences, labels) if lbl == 0] + + pos_frac = _in_range_fraction(pos_lengths, amp_length_min, amp_length_max) + neg_frac = _in_range_fraction(neg_lengths, amp_length_min, amp_length_max) + + if neg_frac > 0: + ratio = pos_frac / neg_frac + elif pos_frac > 0: + ratio = float("inf") + else: + ratio = 1.0 + + length_shortcut_likely = ( + pos_frac >= fraction_threshold + and (ratio >= ratio_threshold or neg_frac == 0.0) + ) + + if length_shortcut_likely: + if neg_frac == 0.0: + explanation = ( + f"{pos_frac:.1%} of positives fall in the AMP length range " + f"[{amp_length_min}, {amp_length_max}] while 0.0% of negatives do. " + "Length is a perfect separator — benchmark improvements may reflect " + "the length shortcut, not genuine activity signal." + ) + else: + explanation = ( + f"{pos_frac:.1%} of positives vs {neg_frac:.1%} of negatives " + f"fall in the AMP length range [{amp_length_min}, {amp_length_max}] " + f"(ratio: {ratio:.2f}x, threshold: {ratio_threshold:.1f}x). " + "Benchmark improvements may reflect the length shortcut, " + "not genuine activity signal." + ) + else: + explanation = ( + f"No length shortcut detected: {pos_frac:.1%} of positives vs " + f"{neg_frac:.1%} of negatives in range [{amp_length_min}, {amp_length_max}] " + f"(ratio: {ratio:.2f}x, threshold: {ratio_threshold:.1f}x)." + ) + + return LengthDistributionReport( + n_positives=len(pos_lengths), + n_negatives=len(neg_lengths), + positive_lengths=pos_lengths, + negative_lengths=neg_lengths, + positive_mean_length=_mean([float(x) for x in pos_lengths]), + positive_median_length=_median([float(x) for x in pos_lengths]), + negative_mean_length=_mean([float(x) for x in neg_lengths]), + negative_median_length=_median([float(x) for x in neg_lengths]), + positive_in_amp_range_fraction=pos_frac, + negative_in_amp_range_fraction=neg_frac, + length_ratio=ratio, + length_shortcut_likely=length_shortcut_likely, + shortcut_explanation=explanation, + amp_length_min=amp_length_min, + amp_length_max=amp_length_max, + ratio_threshold=ratio_threshold, + fraction_threshold=fraction_threshold, + ) + + +def format_length_report(report: LengthDistributionReport) -> str: + """Format a LengthDistributionReport as a human-readable string.""" + ratio_str = ( + f"{report.length_ratio:.2f}x" + if report.length_ratio != float("inf") + else "inf (no negatives in range)" + ) + lines = [ + "=== LENGTH DISTRIBUTION REPORT ===", + f"Sequences: {report.n_positives} positives, {report.n_negatives} negatives", + f"AMP length range: [{report.amp_length_min}, {report.amp_length_max}]", + "", + "-- POSITIVE (AMP) LENGTH STATS --", + f" Mean: {report.positive_mean_length:.2f}", + f" Median: {report.positive_median_length:.2f}", + f" Fraction in AMP range: {report.positive_in_amp_range_fraction:.1%}", + "", + "-- NEGATIVE (non-AMP) LENGTH STATS --", + f" Mean: {report.negative_mean_length:.2f}", + f" Median: {report.negative_median_length:.2f}", + f" Fraction in AMP range: {report.negative_in_amp_range_fraction:.1%}", + "", + f"Pos/neg in-range ratio: {ratio_str}", + f"LENGTH SHORTCUT LIKELY: {'YES -- see explanation below' if report.length_shortcut_likely else 'no'}", + "", + report.shortcut_explanation, + "", + "NOTICE: Length is one of several cheap-explanation dimensions.", + "Combine with charge and similarity reports for full shortcut audit.", + ] + return "\n".join(lines) diff --git a/tests/evidence/test_length_distribution_report.py b/tests/evidence/test_length_distribution_report.py new file mode 100644 index 00000000..54739c8c --- /dev/null +++ b/tests/evidence/test_length_distribution_report.py @@ -0,0 +1,319 @@ +"""Tests for length_distribution_report module (C5).""" +import pytest + +from openamp_foundry.evidence.length_distribution_report import ( + AMP_LENGTH_MAX, + AMP_LENGTH_MIN, + SHORTCUT_FRACTION_THRESHOLD, + SHORTCUT_RATIO_THRESHOLD, + LengthDistributionReport, + _in_range_fraction, + _mean, + _median, + compute_length_report, + format_length_report, +) + + +# --- Constants --- + +class TestConstants: + def test_amp_length_min_is_10(self): + assert AMP_LENGTH_MIN == 10 + + def test_amp_length_max_is_40(self): + assert AMP_LENGTH_MAX == 40 + + def test_shortcut_ratio_threshold(self): + assert SHORTCUT_RATIO_THRESHOLD == pytest.approx(1.5) + + def test_shortcut_fraction_threshold(self): + assert SHORTCUT_FRACTION_THRESHOLD == pytest.approx(0.60) + + +# --- _mean --- + +class TestMean: + def test_empty(self): + assert _mean([]) == pytest.approx(0.0) + + def test_single(self): + assert _mean([5.0]) == pytest.approx(5.0) + + def test_multiple(self): + assert _mean([10.0, 20.0, 30.0]) == pytest.approx(20.0) + + def test_all_same(self): + assert _mean([15.0, 15.0, 15.0]) == pytest.approx(15.0) + + +# --- _median --- + +class TestMedian: + def test_empty(self): + assert _median([]) == pytest.approx(0.0) + + def test_single(self): + assert _median([25.0]) == pytest.approx(25.0) + + def test_odd(self): + assert _median([10.0, 20.0, 30.0]) == pytest.approx(20.0) + + def test_even(self): + assert _median([10.0, 20.0, 30.0, 40.0]) == pytest.approx(25.0) + + def test_unsorted(self): + assert _median([30.0, 10.0, 20.0]) == pytest.approx(20.0) + + +# --- _in_range_fraction --- + +class TestInRangeFraction: + def test_empty(self): + assert _in_range_fraction([], 10, 40) == pytest.approx(0.0) + + def test_all_in_range(self): + assert _in_range_fraction([10, 20, 30, 40], 10, 40) == pytest.approx(1.0) + + def test_none_in_range(self): + assert _in_range_fraction([5, 50, 100], 10, 40) == pytest.approx(0.0) + + def test_half_in_range(self): + assert _in_range_fraction([10, 50], 10, 40) == pytest.approx(0.5) + + def test_boundary_min_included(self): + assert _in_range_fraction([10], 10, 40) == pytest.approx(1.0) + + def test_boundary_max_included(self): + assert _in_range_fraction([40], 10, 40) == pytest.approx(1.0) + + def test_just_below_min_excluded(self): + assert _in_range_fraction([9], 10, 40) == pytest.approx(0.0) + + def test_just_above_max_excluded(self): + assert _in_range_fraction([41], 10, 40) == pytest.approx(0.0) + + +# --- compute_length_report --- + +class TestComputeLengthReport: + def _amp_seq(self, length: int) -> str: + return "A" * length + + def test_no_shortcut_balanced(self): + # Equal fraction of pos and neg in range → no shortcut + seqs = ["A" * 20, "A" * 20, "A" * 20, "A" * 20] + labels = [1, 1, 0, 0] + report = compute_length_report(seqs, labels) + assert report.length_shortcut_likely is False + + def test_shortcut_detected(self): + # Positives all in range, negatives all out of range + seqs = ["A" * 15, "A" * 25, "A" * 5, "A" * 60] + labels = [1, 1, 0, 0] + report = compute_length_report(seqs, labels) + assert report.positive_in_amp_range_fraction == pytest.approx(1.0) + assert report.negative_in_amp_range_fraction == pytest.approx(0.0) + assert report.length_shortcut_likely is True + + def test_length_mismatch_raises(self): + with pytest.raises(ValueError, match="same length"): + compute_length_report(["AA", "KK"], [1]) + + def test_empty_sequences(self): + report = compute_length_report([], []) + assert report.n_positives == 0 + assert report.n_negatives == 0 + assert report.length_shortcut_likely is False + + def test_all_positives(self): + seqs = ["A" * 20, "A" * 25] + labels = [1, 1] + report = compute_length_report(seqs, labels) + assert report.n_negatives == 0 + assert report.n_positives == 2 + + def test_all_negatives(self): + seqs = ["A" * 20, "A" * 25] + labels = [0, 0] + report = compute_length_report(seqs, labels) + assert report.n_positives == 0 + assert report.n_negatives == 2 + + def test_positive_mean_length(self): + seqs = ["A" * 10, "A" * 20, "K" * 5] + labels = [1, 1, 0] + report = compute_length_report(seqs, labels) + assert report.positive_mean_length == pytest.approx(15.0) + + def test_negative_mean_length(self): + seqs = ["A" * 5, "K" * 5] + labels = [0, 0] + report = compute_length_report(seqs, labels) + assert report.negative_mean_length == pytest.approx(5.0) + + def test_positive_median_length(self): + seqs = ["A" * 10, "A" * 20, "A" * 30] + labels = [1, 1, 1] + report = compute_length_report(seqs, labels) + assert report.positive_median_length == pytest.approx(20.0) + + def test_length_ratio_correct(self): + # pos all in range (frac=1.0), neg half in range (frac=0.5) → ratio=2.0 + seqs = ["A" * 20, "A" * 5, "A" * 25] + labels = [1, 0, 0] + report = compute_length_report(seqs, labels) + assert report.positive_in_amp_range_fraction == pytest.approx(1.0) + assert report.negative_in_amp_range_fraction == pytest.approx(0.5) + assert report.length_ratio == pytest.approx(2.0) + + def test_no_neg_in_range_gives_inf_ratio(self): + seqs = ["A" * 20, "A" * 5] + labels = [1, 0] + report = compute_length_report(seqs, labels) + assert report.length_ratio == float("inf") + + def test_no_pos_in_range_no_shortcut(self): + seqs = ["A" * 5, "A" * 20] + labels = [1, 0] + report = compute_length_report(seqs, labels) + assert report.positive_in_amp_range_fraction == pytest.approx(0.0) + assert report.length_shortcut_likely is False + + def test_positive_lengths_list(self): + seqs = ["AA", "AAAA", "K"] + labels = [1, 1, 0] + report = compute_length_report(seqs, labels) + assert sorted(report.positive_lengths) == [2, 4] + + def test_negative_lengths_list(self): + seqs = ["K", "KKK"] + labels = [0, 0] + report = compute_length_report(seqs, labels) + assert sorted(report.negative_lengths) == [1, 3] + + def test_custom_amp_range(self): + seqs = ["A" * 5, "A" * 50] + labels = [1, 0] + report = compute_length_report(seqs, labels, amp_length_min=1, amp_length_max=10) + assert report.positive_in_amp_range_fraction == pytest.approx(1.0) + assert report.negative_in_amp_range_fraction == pytest.approx(0.0) + + def test_thresholds_stored_in_report(self): + report = compute_length_report([], [], ratio_threshold=2.0, fraction_threshold=0.7) + assert report.ratio_threshold == pytest.approx(2.0) + assert report.fraction_threshold == pytest.approx(0.7) + + def test_amp_range_stored_in_report(self): + report = compute_length_report([], [], amp_length_min=5, amp_length_max=50) + assert report.amp_length_min == 5 + assert report.amp_length_max == 50 + + def test_returns_dataclass(self): + report = compute_length_report([], []) + assert isinstance(report, LengthDistributionReport) + + def test_shortcut_explanation_contains_fractions(self): + seqs = ["A" * 20, "A" * 5] + labels = [1, 0] + report = compute_length_report(seqs, labels) + assert "%" in report.shortcut_explanation + + def test_no_shortcut_explanation_format(self): + seqs = ["A" * 20, "A" * 20] + labels = [1, 0] + report = compute_length_report(seqs, labels) + assert "No length shortcut detected" in report.shortcut_explanation + + def test_fraction_threshold_required_for_shortcut(self): + # High ratio but low positive fraction → no shortcut + # pos: 1 seq in range out of 10 → 0.10 (below 0.60 threshold) + pos_seqs = ["A" * 20] + ["A" * 5] * 9 # 1 in range, 9 out + neg_seqs = [] + seqs = pos_seqs + neg_seqs + labels = [1] * 10 + report = compute_length_report(seqs, labels) + assert report.positive_in_amp_range_fraction == pytest.approx(0.1) + assert report.length_shortcut_likely is False + + def test_shortcut_detection_with_perfect_separation(self): + pos_seqs = ["A" * 20, "A" * 25, "A" * 15] + neg_seqs = ["A" * 5, "A" * 60, "A" * 80] + seqs = pos_seqs + neg_seqs + labels = [1, 1, 1, 0, 0, 0] + report = compute_length_report(seqs, labels) + assert report.positive_in_amp_range_fraction == pytest.approx(1.0) + assert report.negative_in_amp_range_fraction == pytest.approx(0.0) + assert report.length_shortcut_likely is True + + def test_zero_everything_is_safe(self): + report = compute_length_report([], []) + assert report.length_ratio == pytest.approx(1.0) + assert report.length_shortcut_likely is False + + +# --- format_length_report --- + +class TestFormatLengthReport: + def _basic_report(self) -> LengthDistributionReport: + seqs = ["A" * 20, "A" * 25, "A" * 5, "A" * 60] + labels = [1, 1, 0, 0] + return compute_length_report(seqs, labels) + + def test_returns_string(self): + assert isinstance(format_length_report(self._basic_report()), str) + + def test_contains_header(self): + assert "LENGTH DISTRIBUTION REPORT" in format_length_report(self._basic_report()) + + def test_contains_positive_count(self): + assert "2 positives" in format_length_report(self._basic_report()) + + def test_contains_negative_count(self): + assert "2 negatives" in format_length_report(self._basic_report()) + + def test_contains_amp_range(self): + text = format_length_report(self._basic_report()) + assert "10" in text and "40" in text + + def test_contains_mean(self): + assert "Mean:" in format_length_report(self._basic_report()) + + def test_contains_median(self): + assert "Median:" in format_length_report(self._basic_report()) + + def test_shortcut_yes_in_text(self): + seqs = ["A" * 20, "A" * 5] + labels = [1, 0] + report = compute_length_report(seqs, labels) + text = format_length_report(report) + assert "YES" in text + + def test_shortcut_no_in_text(self): + seqs = ["A" * 20, "A" * 20] + labels = [1, 0] + report = compute_length_report(seqs, labels) + text = format_length_report(report) + assert "no" in text.lower() + + def test_contains_notice(self): + assert "NOTICE" in format_length_report(self._basic_report()) + + def test_contains_shortcut_explanation(self): + report = self._basic_report() + text = format_length_report(report) + assert report.shortcut_explanation in text + + def test_empty_report_does_not_crash(self): + report = compute_length_report([], []) + text = format_length_report(report) + assert isinstance(text, str) and len(text) > 0 + + def test_positive_section_present(self): + assert "POSITIVE" in format_length_report(self._basic_report()) + + def test_negative_section_present(self): + assert "NEGATIVE" in format_length_report(self._basic_report()) + + def test_ratio_in_output(self): + assert "ratio" in format_length_report(self._basic_report()).lower() diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 366a7224..bca45fd4 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -3,7 +3,7 @@ import sys -BASELINE = 7024 +BASELINE = 7087 TOLERANCE = 0.05