From cf5ed61929270a910b055d749d246ec6be2846bd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:01:32 +0530
Subject: [PATCH 001/439] feat: add normalized analysis findings

---
 src/framevitals/findings.py | 117 ++++++++++++++++++++++++++++++++++++
 1 file changed, 117 insertions(+)
 create mode 100644 src/framevitals/findings.py

diff --git a/src/framevitals/findings.py b/src/framevitals/findings.py
new file mode 100644
index 0000000..a54d990
--- /dev/null
+++ b/src/framevitals/findings.py
@@ -0,0 +1,117 @@
+"""Normalized findings for FrameVitals analysis results.
+
+The current pipeline already produces display-oriented signals with evidence and
+recommendations.  This module converts those signals into a stable, machine-
+readable finding shape without changing any analysis thresholds.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Iterable
+
+
+_SEVERITY_MAP = {
+    "high": "high",
+    "critical": "critical",
+    "medium": "medium",
+    "moderate": "medium",
+    "low": "low",
+    "informational": "info",
+    "info": "info",
+    "none": "none",
+}
+
+_SEVERITY_ORDER = {
+    "critical": 0,
+    "high": 1,
+    "medium": 2,
+    "low": 3,
+    "info": 4,
+    "none": 5,
+}
+
+
+def _slug(value: str) -> str:
+    normalized = re.sub(r"[^a-z0-9]+", "_", value.strip().lower())
+    return normalized.strip("_") or "finding"
+
+
+def normalize_severity(value: Any) -> str:
+    """Return a stable lower-case severity label."""
+    if value is None:
+        return "none"
+    text = str(value).strip().lower()
+    return _SEVERITY_MAP.get(text, text or "none")
+
+
+def _is_actionable_signal(signal: dict[str, Any]) -> bool:
+    """Return whether an existing display signal represents an issue/review.
+
+    Positive informational signals such as "Temporal Data: Detected" remain in
+    ``signals`` but do not become findings.  ML readiness is included whenever
+    it is not fully Ready because its existing signal does not use the generic
+    ``Review`` status.
+    """
+    status = str(signal.get("status", "")).strip().lower()
+    name = str(signal.get("name", "")).strip().lower()
+
+    if status == "review":
+        return True
+    if name == "ml readiness" and status not in {"ready", "good"}:
+        return True
+    return False
+
+
+def findings_from_signals(signals: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Normalize existing pipeline signals into actionable findings.
+
+    No diagnostic logic is introduced here: the function only translates
+    already-computed signal status/evidence/recommendations into a consistent
+    schema for library consumers and report renderers.
+    """
+    findings: list[dict[str, Any]] = []
+
+    for signal in signals:
+        if not isinstance(signal, dict) or not _is_actionable_signal(signal):
+            continue
+
+        name = str(signal.get("name") or "Finding")
+        severity = normalize_severity(signal.get("severity"))
+        finding = {
+            "code": f"signal.{_slug(name)}",
+            "title": name,
+            "severity": severity,
+            "scope": "dataset",
+            "status": signal.get("status"),
+            "evidence": signal.get("evidence") or "",
+            "recommendation": signal.get("recommendation") or "",
+            "method": "signal_engine",
+            "confidence": "deterministic",
+        }
+        findings.append(finding)
+
+    findings.sort(
+        key=lambda item: (
+            _SEVERITY_ORDER.get(str(item.get("severity")), 99),
+            str(item.get("code", "")),
+        )
+    )
+    return findings
+
+
+def recommendations_from_findings(
+    findings: Iterable[dict[str, Any]],
+) -> list[str]:
+    """Return de-duplicated actionable recommendations in finding order."""
+    recommendations: list[str] = []
+    seen: set[str] = set()
+
+    for finding in findings:
+        recommendation = str(finding.get("recommendation") or "").strip()
+        if not recommendation or recommendation in seen:
+            continue
+        seen.add(recommendation)
+        recommendations.append(recommendation)
+
+    return recommendations

From 9b8908d9d2d8db9caff736cddcfeeba06ae5d05c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:01:54 +0530
Subject: [PATCH 002/439] feat: add backward-compatible AnalysisResult

---
 src/framevitals/result.py | 203 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 203 insertions(+)
 create mode 100644 src/framevitals/result.py

diff --git a/src/framevitals/result.py b/src/framevitals/result.py
new file mode 100644
index 0000000..64e8362
--- /dev/null
+++ b/src/framevitals/result.py
@@ -0,0 +1,203 @@
+"""Public result objects for FrameVitals.
+
+``AnalysisResult`` intentionally subclasses :class:`dict` during the 0.x series.
+That preserves existing mapping behaviour and JSON compatibility while adding a
+more discoverable object-oriented API for notebooks, applications, and reports.
+"""
+
+from __future__ import annotations
+
+import json
+from copy import deepcopy
+from pathlib import Path
+from typing import Any
+
+from framevitals.findings import recommendations_from_findings
+
+
+class ColumnResult(dict):
+    """Structured view of one analyzed column.
+
+    The object is also a normal dictionary, making it easy to serialize or pass
+    to existing code while exposing common fields through attribute access.
+    """
+
+    def __getattr__(self, name: str) -> Any:
+        try:
+            return self[name]
+        except KeyError as exc:
+            raise AttributeError(name) from exc
+
+
+class AnalysisResult(dict):
+    """Backward-compatible result returned by :func:`framevitals.analyze`.
+
+    Existing code can continue using dictionary access::
+
+        result["health"]["overall_score"]
+
+    while newer code can use helpers such as ``summary()``, ``column()``,
+    ``to_json()``, ``to_html()``, and ``summary_text()``.
+    """
+
+    schema_version = "1"
+
+    def __getattr__(self, name: str) -> Any:
+        try:
+            return self[name]
+        except KeyError as exc:
+            raise AttributeError(name) from exc
+
+    @property
+    def findings(self) -> list[dict[str, Any]]:
+        value = self.get("findings", [])
+        return value if isinstance(value, list) else []
+
+    @property
+    def recommendations(self) -> list[str]:
+        return recommendations_from_findings(self.findings)
+
+    @property
+    def health(self) -> dict[str, Any]:
+        value = self.get("health", {})
+        return value if isinstance(value, dict) else {}
+
+    @property
+    def ml_readiness(self) -> dict[str, Any]:
+        value = self.get("ml_readiness", {})
+        return value if isinstance(value, dict) else {}
+
+    @property
+    def shape(self) -> dict[str, Any]:
+        profile = self.get("profile", {})
+        if not isinstance(profile, dict):
+            return {}
+        shape = profile.get("shape", {})
+        return shape if isinstance(shape, dict) else {}
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a detached plain-dictionary copy of the complete result."""
+        return deepcopy(dict(self))
+
+    def summary(self) -> dict[str, Any]:
+        """Return a concise, stable high-level summary of the analysis."""
+        health = self.health
+        ml_readiness = self.ml_readiness
+        timings = self.get("timings_ms", {})
+        if not isinstance(timings, dict):
+            timings = {}
+
+        severity_counts: dict[str, int] = {}
+        for finding in self.findings:
+            severity = str(finding.get("severity") or "unknown")
+            severity_counts[severity] = severity_counts.get(severity, 0) + 1
+
+        return {
+            "dataset_id": self.get("dataset_id"),
+            "filename": self.get("filename"),
+            "analysis_mode": self.get("analysis_mode"),
+            "shape": dict(self.shape),
+            "health": {
+                "overall_score": health.get("overall_score"),
+                "label": health.get("label"),
+            },
+            "ml_readiness": {
+                "score": ml_readiness.get("score"),
+                "label": ml_readiness.get("label"),
+            },
+            "finding_count": len(self.findings),
+            "finding_severity_counts": severity_counts,
+            "artifacts_enabled": bool(self.get("artifacts_enabled", False)),
+            "total_ms": timings.get("total"),
+        }
+
+    def column(self, name: str) -> ColumnResult:
+        """Return the combined profile/role view for a single column.
+
+        Raises
+        ------
+        KeyError
+            If the requested column was not part of the analyzed dataset.
+        """
+        profile = self.get("profile", {})
+        roles = self.get("column_roles", {})
+        if not isinstance(profile, dict):
+            profile = {}
+        if not isinstance(roles, dict):
+            roles = {}
+
+        columns = profile.get("columns", [])
+        if name not in columns and name not in roles:
+            raise KeyError(f"Column not found in analysis result: {name}")
+
+        role_info = roles.get(name, {})
+        if not isinstance(role_info, dict):
+            role_info = {}
+
+        numeric_summary = profile.get("numeric_summary", {})
+        categorical_summary = profile.get("categorical_summary", {})
+        correlations = profile.get("correlations", {})
+        if not isinstance(numeric_summary, dict):
+            numeric_summary = {}
+        if not isinstance(categorical_summary, dict):
+            categorical_summary = {}
+        if not isinstance(correlations, dict):
+            correlations = {}
+
+        payload = {
+            "name": name,
+            "dtype": profile.get("dtypes", {}).get(name),
+            "roles": list(role_info.get("roles", [])),
+            "missing_count": profile.get("missing_counts", {}).get(name),
+            "missing_percent": profile.get("missing_percent", {}).get(name),
+            "unique_count": role_info.get("unique_count"),
+            "unique_ratio": role_info.get("unique_ratio"),
+            "non_missing_count": role_info.get("non_missing_count"),
+            "is_numeric": role_info.get("is_numeric"),
+            "is_categorical": role_info.get("is_categorical"),
+            "numeric_summary": numeric_summary.get(name),
+            "categorical_summary": categorical_summary.get(name),
+            "correlations": correlations.get(name, {}),
+        }
+        return ColumnResult(payload)
+
+    def to_json(
+        self,
+        destination: str | Path | None = None,
+        *,
+        indent: int = 2,
+    ) -> str | Path:
+        """Serialize the complete result to JSON or write it to ``destination``."""
+        rendered = json.dumps(self.to_dict(), indent=indent, default=str)
+        if destination is None:
+            return rendered
+
+        path = Path(destination)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(rendered + "\n", encoding="utf-8")
+        return path
+
+    def summary_text(self) -> str:
+        """Render a compact human-readable terminal summary."""
+        from framevitals.reporting.terminal import render_terminal_summary
+
+        return render_terminal_summary(self)
+
+    def to_html(self, destination: str | Path | None = None) -> str | Path:
+        """Render a self-contained HTML report and optionally write it to disk."""
+        from framevitals.reporting.html import render_html_report
+
+        rendered = render_html_report(self)
+        if destination is None:
+            return rendered
+
+        path = Path(destination)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(rendered, encoding="utf-8")
+        return path
+
+    def _repr_html_(self) -> str:
+        """Provide a compact rich representation in Jupyter-compatible clients."""
+        from framevitals.reporting.html import render_notebook_summary
+
+        return render_notebook_summary(self)

From a1c368347aafc084d22f8f4f232425c3586249fd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:02:05 +0530
Subject: [PATCH 003/439] feat: add reporting package

---
 src/framevitals/reporting/__init__.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 src/framevitals/reporting/__init__.py

diff --git a/src/framevitals/reporting/__init__.py b/src/framevitals/reporting/__init__.py
new file mode 100644
index 0000000..102ce01
--- /dev/null
+++ b/src/framevitals/reporting/__init__.py
@@ -0,0 +1,14 @@
+"""Human-facing renderers for FrameVitals results.
+
+Renderers are intentionally dependency-light and consume the stable result
+shape rather than re-running analysis.
+"""
+
+from framevitals.reporting.html import render_html_report, render_notebook_summary
+from framevitals.reporting.terminal import render_terminal_summary
+
+__all__ = [
+    "render_html_report",
+    "render_notebook_summary",
+    "render_terminal_summary",
+]

From 5254f2e7d1202676d76e24026a44a2f057f6c304 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:02:16 +0530
Subject: [PATCH 004/439] feat: add terminal result renderer

---
 src/framevitals/reporting/terminal.py | 80 +++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 src/framevitals/reporting/terminal.py

diff --git a/src/framevitals/reporting/terminal.py b/src/framevitals/reporting/terminal.py
new file mode 100644
index 0000000..b8f61ad
--- /dev/null
+++ b/src/framevitals/reporting/terminal.py
@@ -0,0 +1,80 @@
+"""Dependency-light terminal rendering for FrameVitals results."""
+
+from __future__ import annotations
+
+from typing import Any, Mapping
+
+
+def _score_bar(value: Any, width: int = 24) -> str:
+    try:
+        score = max(0.0, min(100.0, float(value)))
+    except (TypeError, ValueError):
+        return "[" + "?" * width + "]"
+    filled = round(score / 100 * width)
+    return "[" + "#" * filled + "-" * (width - filled) + "]"
+
+
+def _fmt_score(value: Any) -> str:
+    try:
+        return f"{float(value):.1f}/100"
+    except (TypeError, ValueError):
+        return "n/a"
+
+
+def _clean_line(value: Any, *, max_length: int = 110) -> str:
+    text = " ".join(str(value or "").split())
+    if len(text) <= max_length:
+        return text
+    return text[: max_length - 1].rstrip() + "…"
+
+
+def render_terminal_summary(result: Mapping[str, Any]) -> str:
+    """Render a compact report suitable for interactive terminal output."""
+    profile = result.get("profile", {}) or {}
+    shape = profile.get("shape", {}) or {}
+    health = result.get("health", {}) or {}
+    ml = result.get("ml_readiness", {}) or {}
+    findings = result.get("findings", []) or []
+    timings = result.get("timings_ms", {}) or {}
+
+    rows = shape.get("rows", "?")
+    columns = shape.get("columns", "?")
+    health_score = health.get("overall_score")
+    ml_score = ml.get("score")
+
+    lines = [
+        "FrameVitals Analysis",
+        "=" * 72,
+        f"Dataset       {result.get('filename', '<unknown>')}",
+        f"Mode          {result.get('analysis_mode', 'unknown')}",
+        f"Shape         {rows} rows x {columns} columns",
+        f"Memory        {profile.get('memory_usage_mb', 'n/a')} MB",
+        "",
+        f"Health        {_score_bar(health_score)}  {_fmt_score(health_score)}  {health.get('label', '')}",
+        f"ML readiness  {_score_bar(ml_score)}  {_fmt_score(ml_score)}  {ml.get('label', '')}",
+        "",
+        f"Findings      {len(findings)} actionable issue(s)",
+    ]
+
+    if findings:
+        for finding in findings[:6]:
+            severity = str(finding.get("severity", "info")).upper()
+            title = _clean_line(finding.get("title", "Finding"), max_length=42)
+            evidence = _clean_line(finding.get("evidence", ""), max_length=88)
+            lines.append(f"  [{severity:<8}] {title}")
+            if evidence:
+                lines.append(f"             {evidence}")
+        if len(findings) > 6:
+            lines.append(f"  ... and {len(findings) - 6} more finding(s)")
+    else:
+        lines.append("  No actionable findings from the current signal layer.")
+
+    total_ms = timings.get("total")
+    if isinstance(total_ms, (int, float)):
+        lines.extend(["", f"Completed in   {total_ms / 1000:.2f}s"])
+
+    lines.extend([
+        "=" * 72,
+        "Use result.to_html(...) or CLI --output to keep the complete report.",
+    ])
+    return "\n".join(lines)

From 1625d6555a5db192723f159af73aef4a76953a1d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:02:47 +0530
Subject: [PATCH 005/439] feat: add standalone HTML report renderer

---
 src/framevitals/reporting/html.py | 217 ++++++++++++++++++++++++++++++
 1 file changed, 217 insertions(+)
 create mode 100644 src/framevitals/reporting/html.py

diff --git a/src/framevitals/reporting/html.py b/src/framevitals/reporting/html.py
new file mode 100644
index 0000000..85d1644
--- /dev/null
+++ b/src/framevitals/reporting/html.py
@@ -0,0 +1,217 @@
+"""Self-contained HTML rendering for FrameVitals analysis results."""
+
+from __future__ import annotations
+
+import json
+from html import escape
+from typing import Any, Mapping
+
+
+def _text(value: Any) -> str:
+    return escape(str(value if value is not None else ""))
+
+
+def _score(value: Any) -> float:
+    try:
+        return max(0.0, min(100.0, float(value)))
+    except (TypeError, ValueError):
+        return 0.0
+
+
+def _severity_class(value: Any) -> str:
+    severity = str(value or "info").lower()
+    if severity in {"critical", "high", "medium", "low", "info"}:
+        return severity
+    return "info"
+
+
+def _column_rows(result: Mapping[str, Any]) -> str:
+    profile = result.get("profile", {}) or {}
+    roles = result.get("column_roles", {}) or {}
+    columns = profile.get("columns", []) or []
+    dtypes = profile.get("dtypes", {}) or {}
+    missing = profile.get("missing_percent", {}) or {}
+
+    rows: list[str] = []
+    for name in columns:
+        role_info = roles.get(name, {}) or {}
+        role_text = ", ".join(role_info.get("roles", [])[:6])
+        unique_count = role_info.get("unique_count", "")
+        rows.append(
+            "<tr>"
+            f"<td><code>{_text(name)}</code></td>"
+            f"<td>{_text(dtypes.get(name, ''))}</td>"
+            f"<td>{_text(missing.get(name, ''))}%</td>"
+            f"<td>{_text(unique_count)}</td>"
+            f"<td>{_text(role_text)}</td>"
+            "</tr>"
+        )
+    return "".join(rows)
+
+
+def _finding_cards(result: Mapping[str, Any]) -> str:
+    findings = result.get("findings", []) or []
+    if not findings:
+        return '<div class="empty">No actionable findings from the current signal layer.</div>'
+
+    cards: list[str] = []
+    for finding in findings:
+        severity = _severity_class(finding.get("severity"))
+        recommendation = finding.get("recommendation") or ""
+        recommendation_html = (
+            f'<div class="recommendation"><strong>Next action</strong><br>{_text(recommendation)}</div>'
+            if recommendation
+            else ""
+        )
+        cards.append(
+            f'<article class="finding {severity}">'
+            f'<div class="finding-top"><span class="badge">{_text(severity.upper())}</span>'
+            f'<span class="finding-code">{_text(finding.get("code", ""))}</span></div>'
+            f'<h3>{_text(finding.get("title", "Finding"))}</h3>'
+            f'<p>{_text(finding.get("evidence", ""))}</p>'
+            f'{recommendation_html}'
+            '</article>'
+        )
+    return "".join(cards)
+
+
+def render_notebook_summary(result: Mapping[str, Any]) -> str:
+    """Return a compact notebook-safe HTML representation."""
+    profile = result.get("profile", {}) or {}
+    shape = profile.get("shape", {}) or {}
+    health = result.get("health", {}) or {}
+    ml = result.get("ml_readiness", {}) or {}
+    findings = result.get("findings", []) or []
+    return f"""
+<div style="font-family:Inter,system-ui,-apple-system,sans-serif;border:1px solid #e5e7eb;border-radius:14px;padding:16px 18px;max-width:760px;background:#fff;color:#111827">
+  <div style="display:flex;justify-content:space-between;gap:16px;align-items:center;margin-bottom:12px">
+    <div><strong style="font-size:18px">FrameVitals</strong><div style="color:#6b7280;font-size:13px">{_text(result.get('filename', '<dataframe>'))}</div></div>
+    <div style="font-size:13px;color:#6b7280">{_text(shape.get('rows','?'))} rows × {_text(shape.get('columns','?'))} columns</div>
+  </div>
+  <div style="display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:10px">
+    <div style="background:#f9fafb;padding:12px;border-radius:10px"><div style="font-size:12px;color:#6b7280">Health</div><strong>{_text(health.get('overall_score','n/a'))}/100</strong><div style="font-size:12px">{_text(health.get('label',''))}</div></div>
+    <div style="background:#f9fafb;padding:12px;border-radius:10px"><div style="font-size:12px;color:#6b7280">ML readiness</div><strong>{_text(ml.get('score','n/a'))}/100</strong><div style="font-size:12px">{_text(ml.get('label',''))}</div></div>
+    <div style="background:#f9fafb;padding:12px;border-radius:10px"><div style="font-size:12px;color:#6b7280">Findings</div><strong>{len(findings)}</strong><div style="font-size:12px">actionable</div></div>
+  </div>
+</div>
+""".strip()
+
+
+def render_html_report(result: Mapping[str, Any]) -> str:
+    """Render a complete, self-contained HTML analysis report."""
+    profile = result.get("profile", {}) or {}
+    shape = profile.get("shape", {}) or {}
+    health = result.get("health", {}) or {}
+    ml = result.get("ml_readiness", {}) or {}
+    findings = result.get("findings", []) or []
+    recommendations = []
+    seen: set[str] = set()
+    for finding in findings:
+        recommendation = str(finding.get("recommendation") or "").strip()
+        if recommendation and recommendation not in seen:
+            seen.add(recommendation)
+            recommendations.append(recommendation)
+
+    health_score = _score(health.get("overall_score"))
+    ml_score = _score(ml.get("score"))
+    missing_percent = health.get("details", {}).get("missing_percent", 0)
+    duplicate_percent = profile.get("duplicate_percent", 0)
+    memory = profile.get("memory_usage_mb", "n/a")
+    total_ms = (result.get("timings_ms", {}) or {}).get("total")
+    duration = f"{float(total_ms) / 1000:.2f}s" if isinstance(total_ms, (int, float)) else "n/a"
+
+    recommendation_items = "".join(
+        f"<li>{_text(item)}</li>" for item in recommendations
+    ) or "<li>No additional remediation steps are required by the current finding layer.</li>"
+
+    raw_json = escape(json.dumps(dict(result), indent=2, default=str))
+
+    return f"""<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width,initial-scale=1">
+<title>FrameVitals Report — {_text(result.get('filename', 'dataset'))}</title>
+<style>
+:root {{ color-scheme: light; --bg:#f5f7fb; --panel:#ffffff; --ink:#111827; --muted:#687386; --line:#e6eaf0; --accent:#3157d5; --good:#15803d; --warn:#b45309; --bad:#b42318; }}
+* {{ box-sizing:border-box; }}
+body {{ margin:0;background:var(--bg);color:var(--ink);font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;line-height:1.5; }}
+.wrap {{ max-width:1180px;margin:0 auto;padding:42px 22px 70px; }}
+.hero {{ background:linear-gradient(135deg,#111827 0%,#1f2a44 62%,#3157d5 100%);color:white;border-radius:24px;padding:34px;box-shadow:0 18px 60px rgba(17,24,39,.16); }}
+.eyebrow {{ font-size:12px;font-weight:800;letter-spacing:.14em;text-transform:uppercase;opacity:.72; }}
+h1 {{ margin:7px 0 6px;font-size:clamp(32px,5vw,58px);line-height:1.04; }}
+.hero p {{ margin:0;color:#d7ddec; }}
+.meta {{ display:flex;gap:10px;flex-wrap:wrap;margin-top:22px; }}
+.meta span {{ padding:7px 10px;border:1px solid rgba(255,255,255,.18);border-radius:999px;background:rgba(255,255,255,.07);font-size:13px; }}
+.grid {{ display:grid;grid-template-columns:repeat(12,1fr);gap:16px;margin-top:18px; }}
+.panel {{ background:var(--panel);border:1px solid var(--line);border-radius:18px;padding:20px;box-shadow:0 5px 18px rgba(17,24,39,.035); }}
+.score {{ grid-column:span 4;min-height:170px; }}
+.metric {{ grid-column:span 3; }}
+.section {{ grid-column:1/-1; }}
+.label {{ color:var(--muted);font-size:12px;font-weight:800;letter-spacing:.08em;text-transform:uppercase; }}
+.big {{ font-size:36px;font-weight:800;margin-top:7px; }}
+.bar {{ height:9px;background:#edf0f5;border-radius:999px;overflow:hidden;margin:18px 0 9px; }}
+.bar > span {{ display:block;height:100%;border-radius:inherit;background:linear-gradient(90deg,#3157d5,#6d8aff); }}
+.findings {{ display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:12px;margin-top:14px; }}
+.finding {{ border:1px solid var(--line);border-left-width:4px;border-radius:14px;padding:16px;background:#fff; }}
+.finding.critical,.finding.high {{ border-left-color:#d92d20; }}
+.finding.medium {{ border-left-color:#f79009; }}
+.finding.low {{ border-left-color:#2e90fa; }}
+.finding.info {{ border-left-color:#667085; }}
+.finding-top {{ display:flex;justify-content:space-between;gap:10px;align-items:center; }}
+.badge {{ font-size:10px;font-weight:900;letter-spacing:.08em;padding:4px 7px;border-radius:999px;background:#f2f4f7; }}
+.finding-code {{ color:var(--muted);font-size:11px;font-family:ui-monospace,SFMono-Regular,Menlo,monospace; }}
+.finding h3 {{ margin:10px 0 5px;font-size:17px; }}
+.finding p {{ margin:0;color:#475467;font-size:14px; }}
+.recommendation {{ margin-top:12px;padding:10px 12px;background:#f8fafc;border-radius:10px;font-size:13px;color:#344054; }}
+table {{ width:100%;border-collapse:collapse;margin-top:12px;font-size:13px; }}
+th,td {{ text-align:left;padding:10px 9px;border-bottom:1px solid var(--line);vertical-align:top; }}
+th {{ color:#475467;font-size:11px;text-transform:uppercase;letter-spacing:.05em; }}
+code {{ font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.95em; }}
+ul {{ padding-left:20px; }}
+details {{ margin-top:12px; }}
+summary {{ cursor:pointer;font-weight:700; }}
+pre {{ overflow:auto;background:#111827;color:#d1d5db;padding:16px;border-radius:12px;font-size:11px;max-height:520px; }}
+.empty {{ color:var(--muted);padding:18px 0; }}
+.footer {{ color:var(--muted);text-align:center;font-size:12px;margin-top:26px; }}
+@media (max-width:820px) {{ .score,.metric {{ grid-column:1/-1; }} .findings {{ grid-template-columns:1fr; }} .hero {{ padding:25px; }} }}
+</style>
+</head>
+<body>
+<main class="wrap">
+<section class="hero">
+  <div class="eyebrow">FrameVitals analysis report</div>
+  <h1>{_text(result.get('filename', '<dataframe>'))}</h1>
+  <p>Data health, structure, ML readiness, and actionable diagnostics in one report.</p>
+  <div class="meta">
+    <span>{_text(shape.get('rows','?'))} rows</span>
+    <span>{_text(shape.get('columns','?'))} columns</span>
+    <span>{_text(result.get('analysis_mode','unknown'))} mode</span>
+    <span>{len(findings)} findings</span>
+    <span>{_text(duration)} runtime</span>
+  </div>
+</section>
+
+<section class="grid">
+  <article class="panel score"><div class="label">Data health</div><div class="big">{_text(health.get('overall_score','n/a'))}<span style="font-size:17px;color:#98a2b3"> / 100</span></div><div>{_text(health.get('label',''))}</div><div class="bar"><span style="width:{health_score:.1f}%"></span></div></article>
+  <article class="panel score"><div class="label">ML readiness</div><div class="big">{_text(ml.get('score','n/a'))}<span style="font-size:17px;color:#98a2b3"> / 100</span></div><div>{_text(ml.get('label',''))}</div><div class="bar"><span style="width:{ml_score:.1f}%"></span></div></article>
+  <article class="panel score"><div class="label">Actionable findings</div><div class="big">{len(findings)}</div><div style="color:#667085">Normalized from FrameVitals' deterministic signal layer.</div></article>
+
+  <article class="panel metric"><div class="label">Missing cells</div><div class="big" style="font-size:27px">{_text(missing_percent)}%</div></article>
+  <article class="panel metric"><div class="label">Duplicate rows</div><div class="big" style="font-size:27px">{_text(duplicate_percent)}%</div></article>
+  <article class="panel metric"><div class="label">Memory</div><div class="big" style="font-size:27px">{_text(memory)} MB</div></article>
+  <article class="panel metric"><div class="label">Runtime</div><div class="big" style="font-size:27px">{_text(duration)}</div></article>
+
+  <section class="panel section"><div class="label">Findings</div><h2>What needs attention</h2><div class="findings">{_finding_cards(result)}</div></section>
+
+  <section class="panel section"><div class="label">Recommendations</div><h2>Suggested next actions</h2><ul>{recommendation_items}</ul></section>
+
+  <section class="panel section"><div class="label">Columns</div><h2>Dataset structure</h2><div style="overflow:auto"><table><thead><tr><th>Column</th><th>dtype</th><th>Missing</th><th>Unique</th><th>Roles</th></tr></thead><tbody>{_column_rows(result)}</tbody></table></div></section>
+
+  <section class="panel section"><div class="label">Complete result</div><details><summary>Inspect raw JSON</summary><pre>{raw_json}</pre></details></section>
+</section>
+<div class="footer">Generated locally by FrameVitals. This HTML file is self-contained and does not require a server.</div>
+</main>
+</body>
+</html>
+"""

From 4520d8d5b5352ae2c19951cc7f23532e930ec3e5 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:03:15 +0530
Subject: [PATCH 006/439] feat: attach schema metadata and findings to results

---
 src/framevitals/result.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/result.py b/src/framevitals/result.py
index 64e8362..99584bc 100644
--- a/src/framevitals/result.py
+++ b/src/framevitals/result.py
@@ -12,7 +12,7 @@
 from pathlib import Path
 from typing import Any
 
-from framevitals.findings import recommendations_from_findings
+from framevitals.findings import findings_from_signals, recommendations_from_findings
 
 
 class ColumnResult(dict):
@@ -42,6 +42,15 @@ class AnalysisResult(dict):
 
     schema_version = "1"
 
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.setdefault("result_schema_version", self.schema_version)
+        if "findings" not in self:
+            signals = self.get("signals", [])
+            if not isinstance(signals, list):
+                signals = []
+            self["findings"] = findings_from_signals(signals)
+
     def __getattr__(self, name: str) -> Any:
         try:
             return self[name]
@@ -96,6 +105,7 @@ def summary(self) -> dict[str, Any]:
             "dataset_id": self.get("dataset_id"),
             "filename": self.get("filename"),
             "analysis_mode": self.get("analysis_mode"),
+            "result_schema_version": self.get("result_schema_version"),
             "shape": dict(self.shape),
             "health": {
                 "overall_score": health.get("overall_score"),

From 0d45a3dc701cd75be620236a4f72c2d5d4a72553 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:03:31 +0530
Subject: [PATCH 007/439] feat: return AnalysisResult from analyze

---
 src/framevitals/api.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index 7b57be0..900e732 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -12,6 +12,7 @@
 from framevitals.drift_analysis import compare_datasets
 from framevitals.loader import load_dataset
 from framevitals.pipeline import run_full_analysis
+from framevitals.result import AnalysisResult
 
 VALID_MODES = {
     "quick",
@@ -56,7 +57,7 @@ def analyze(
     target: str | None = None,
     mode: str = "standard",
     artifacts: bool = False,
-) -> dict:
+) -> AnalysisResult:
     """Analyze a tabular dataset with FrameVitals.
 
     Parameters
@@ -73,16 +74,18 @@ def analyze(
 
     Returns
     -------
-    dict
-        Structured FrameVitals analysis results.
+    AnalysisResult
+        A dict-compatible structured result with summary, column, JSON, HTML,
+        and notebook helpers.
 
     Examples
     --------
     >>> import pandas as pd
     >>> import framevitals as fv
     >>> df = pd.DataFrame({"age": [20, 30], "income": [30000, 50000]})
-    >>> report = fv.analyze(df, mode="quick")
-    >>> print(report["health"]["overall_score"])
+    >>> result = fv.analyze(df, mode="quick")
+    >>> print(result["health"]["overall_score"])
+    >>> print(result.summary())
     """
     if mode not in VALID_MODES:
         raise ValueError(
@@ -95,7 +98,7 @@ def analyze(
     if isinstance(data, pd.DataFrame):
         if data.empty:
             raise ValueError("Dataset DataFrame is empty.")
-        return run_full_analysis(
+        payload = run_full_analysis(
             dataset_id=dataset_id,
             dataframe=data,
             original_filename="<dataframe>",
@@ -104,10 +107,11 @@ def analyze(
             skip_ai=True,
             write_artifacts=artifacts,
         )
+        return AnalysisResult(payload)
 
     if isinstance(data, (str, Path)):
         path = _validated_path(data)
-        return run_full_analysis(
+        payload = run_full_analysis(
             dataset_id=dataset_id,
             file_path=path,
             original_filename=path.name,
@@ -116,6 +120,7 @@ def analyze(
             skip_ai=True,
             write_artifacts=artifacts,
         )
+        return AnalysisResult(payload)
 
     raise TypeError(
         "data must be a pandas DataFrame or a path to a supported dataset."

From 8faa7aefdbf7652b7dcb39a117241e1dddc2c8f0 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:03:42 +0530
Subject: [PATCH 008/439] feat: expose AnalysisResult types

---
 src/framevitals/__init__.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 1ca3332..fdeb0b4 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -4,6 +4,8 @@
 
 from typing import Any
 
+from framevitals.result import AnalysisResult, ColumnResult
+
 __version__ = "0.1.0"
 
 
@@ -13,11 +15,12 @@ def analyze(
     target: str | None = None,
     mode: str = "standard",
     artifacts: bool = False,
-) -> dict[str, Any]:
+) -> AnalysisResult:
     """Analyze a tabular dataset from a DataFrame or supported file path.
 
-    The implementation is imported lazily so ``import framevitals`` and
-    ``framevitals --version`` do not initialize the complete analytics stack.
+    The analytics implementation is imported lazily so ``import framevitals``
+    and ``framevitals --version`` do not initialize the complete analytics
+    stack. The lightweight result types remain available at package level.
     """
     from framevitals.api import analyze as _analyze
 
@@ -61,4 +64,12 @@ def validate(data: Any, contract: dict[str, Any]) -> dict[str, Any]:
     return _validate(data, contract)
 
 
-__all__ = ["analyze", "compare", "infer_contract", "validate", "__version__"]
+__all__ = [
+    "AnalysisResult",
+    "ColumnResult",
+    "analyze",
+    "compare",
+    "infer_contract",
+    "validate",
+    "__version__",
+]

From b4c59019b2454bbeddff551a2f3643bedb7cf6ac Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:04:01 +0530
Subject: [PATCH 009/439] test: cover AnalysisResult and report renderers

---
 tests/test_result_reporting.py | 159 +++++++++++++++++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100644 tests/test_result_reporting.py

diff --git a/tests/test_result_reporting.py b/tests/test_result_reporting.py
new file mode 100644
index 0000000..2acf3a8
--- /dev/null
+++ b/tests/test_result_reporting.py
@@ -0,0 +1,159 @@
+import json
+
+import pandas as pd
+import pytest
+
+import framevitals
+from framevitals.result import AnalysisResult, ColumnResult
+
+
+def _sample_result() -> AnalysisResult:
+    return AnalysisResult({
+        "dataset_id": "fv_test",
+        "filename": "customers.csv",
+        "analysis_mode": "quick",
+        "artifacts_enabled": False,
+        "profile": {
+            "shape": {"rows": 4, "columns": 2},
+            "columns": ["age", "city"],
+            "dtypes": {"age": "float64", "city": "object"},
+            "missing_counts": {"age": 1, "city": 0},
+            "missing_percent": {"age": 25.0, "city": 0.0},
+            "duplicate_rows": 0,
+            "duplicate_percent": 0.0,
+            "memory_usage_mb": 0.01,
+            "numeric_summary": {"age": {"mean": 30.0}},
+            "categorical_summary": {
+                "city": {"unique_values": 2, "top_values": {"Pune": 3}},
+            },
+            "correlations": {},
+        },
+        "column_roles": {
+            "age": {
+                "roles": ["numeric", "analysis_candidate"],
+                "unique_count": 3,
+                "unique_ratio": 0.75,
+                "non_missing_count": 3,
+                "is_numeric": True,
+                "is_categorical": False,
+            },
+            "city": {
+                "roles": ["categorical", "low_cardinality"],
+                "unique_count": 2,
+                "unique_ratio": 0.5,
+                "non_missing_count": 4,
+                "is_numeric": False,
+                "is_categorical": True,
+            },
+        },
+        "health": {
+            "overall_score": 82.5,
+            "label": "Good",
+            "details": {"missing_percent": 12.5},
+        },
+        "ml_readiness": {"score": 72.0, "label": "Mostly Ready"},
+        "signals": [
+            {
+                "name": "Data Completeness",
+                "status": "Review",
+                "severity": "Medium",
+                "evidence": "12.5% of dataset cells are missing.",
+                "recommendation": "Review missing values.",
+            },
+            {
+                "name": "Temporal Data",
+                "status": "Detected",
+                "severity": "Informational",
+                "evidence": "A date column exists.",
+                "recommendation": "Use temporal analysis.",
+            },
+        ],
+        "timings_ms": {"total": 125.0},
+    })
+
+
+def test_analysis_result_remains_dict_compatible():
+    result = _sample_result()
+
+    assert isinstance(result, dict)
+    assert result["health"]["overall_score"] == 82.5
+    assert result.get("filename") == "customers.csv"
+    assert result["result_schema_version"] == "1"
+
+
+def test_findings_are_normalized_from_existing_signals():
+    result = _sample_result()
+
+    assert len(result.findings) == 1
+    finding = result.findings[0]
+    assert finding["code"] == "signal.data_completeness"
+    assert finding["severity"] == "medium"
+    assert finding["method"] == "signal_engine"
+    assert result.recommendations == ["Review missing values."]
+
+
+def test_summary_and_column_helpers():
+    result = _sample_result()
+
+    summary = result.summary()
+    assert summary["shape"] == {"rows": 4, "columns": 2}
+    assert summary["finding_count"] == 1
+    assert summary["health"]["overall_score"] == 82.5
+
+    column = result.column("age")
+    assert isinstance(column, ColumnResult)
+    assert column.name == "age"
+    assert column.missing_percent == 25.0
+    assert column.numeric_summary["mean"] == 30.0
+
+    with pytest.raises(KeyError, match="Column not found"):
+        result.column("does_not_exist")
+
+
+def test_json_export_is_full_and_round_trips(tmp_path):
+    result = _sample_result()
+
+    rendered = result.to_json()
+    payload = json.loads(rendered)
+    assert payload["profile"]["columns"] == ["age", "city"]
+    assert payload["findings"][0]["code"] == "signal.data_completeness"
+
+    destination = tmp_path / "nested" / "report.json"
+    returned = result.to_json(destination)
+    assert returned == destination
+    written = json.loads(destination.read_text(encoding="utf-8"))
+    assert written["dataset_id"] == "fv_test"
+
+
+def test_terminal_and_html_renderers(tmp_path):
+    result = _sample_result()
+
+    terminal = result.summary_text()
+    assert "FrameVitals Analysis" in terminal
+    assert "customers.csv" in terminal
+    assert "Data Completeness" in terminal
+
+    html = result.to_html()
+    assert "<!doctype html>" in html.lower()
+    assert "customers.csv" in html
+    assert "Data Completeness" in html
+    assert "Inspect raw JSON" in html
+
+    destination = tmp_path / "report.html"
+    returned = result.to_html(destination)
+    assert returned == destination
+    assert destination.exists()
+
+
+def test_public_analyze_returns_analysis_result():
+    df = pd.DataFrame({
+        "age": [20, 30, 40, 50],
+        "city": ["Pune", "Mumbai", "Pune", "Nashik"],
+    })
+
+    result = framevitals.analyze(df, mode="quick")
+
+    assert isinstance(result, framevitals.AnalysisResult)
+    assert isinstance(result, dict)
+    assert result["profile"]["shape"]["rows"] == 4
+    assert result.result_schema_version == "1"

From 0677a0cf55ed6cdd2a743f910c14274f8a06093f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:04:41 +0530
Subject: [PATCH 010/439] feat: add full CLI exports and terminal format

---
 src/framevitals/cli.py | 37 ++++++++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/src/framevitals/cli.py b/src/framevitals/cli.py
index ec27df3..0b24008 100644
--- a/src/framevitals/cli.py
+++ b/src/framevitals/cli.py
@@ -10,7 +10,7 @@ def _add_output_argument(parser: argparse.ArgumentParser) -> None:
         "--output",
         type=Path,
         default=None,
-        help="Optional path to write the JSON result.",
+        help="Optional path to write the complete JSON result.",
     )
 
 
@@ -69,6 +69,18 @@ def build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Persist cleaned CSV/chart artifacts.",
     )
+    analyze_parser.add_argument(
+        "--format",
+        choices=["terminal", "json"],
+        default="terminal",
+        help="Stdout format. JSON prints the complete result.",
+    )
+    analyze_parser.add_argument(
+        "--html-report",
+        type=Path,
+        default=None,
+        help="Optional path to write a self-contained HTML report.",
+    )
     _add_output_argument(analyze_parser)
 
     compare_parser = subparsers.add_parser(
@@ -151,16 +163,19 @@ def main() -> int:
             artifacts=args.artifacts,
         )
 
-        summary = {
-            "file": report.get("filename"),
-            "mode": report.get("analysis_mode"),
-            "health": report.get("health"),
-            "ml_readiness": report.get("ml_readiness"),
-            "dataset_signals": report.get("dataset_signals"),
-            "artifacts_enabled": report.get("artifacts_enabled"),
-            "timings_ms": report.get("timings_ms"),
-        }
-        _emit(summary, args.output)
+        if args.output is not None:
+            report.to_json(args.output)
+        if args.html_report is not None:
+            report.to_html(args.html_report)
+
+        if args.format == "json":
+            print(report.to_json())
+        else:
+            print(report.summary_text())
+            if args.output is not None:
+                print(f"Full JSON     {args.output}")
+            if args.html_report is not None:
+                print(f"HTML report   {args.html_report}")
         return 0
 
     if args.command == "compare":

From fac5b6b8e898f43c02dc80e5c115cf40d37a8280 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:04:57 +0530
Subject: [PATCH 011/439] test: cover full CLI JSON and HTML exports

---
 tests/test_cli_reporting.py | 77 +++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 tests/test_cli_reporting.py

diff --git a/tests/test_cli_reporting.py b/tests/test_cli_reporting.py
new file mode 100644
index 0000000..d428de9
--- /dev/null
+++ b/tests/test_cli_reporting.py
@@ -0,0 +1,77 @@
+import json
+
+from framevitals.cli import main
+
+
+def _write_dataset(path):
+    path.write_text(
+        "age,city\n"
+        "20,Pune\n"
+        "30,Mumbai\n"
+        "40,Pune\n"
+        "50,Nashik\n",
+        encoding="utf-8",
+    )
+
+
+def test_analyze_cli_writes_full_json_and_html(tmp_path, monkeypatch, capsys):
+    dataset = tmp_path / "customers.csv"
+    output = tmp_path / "report.json"
+    html = tmp_path / "report.html"
+    _write_dataset(dataset)
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "analyze",
+            str(dataset),
+            "--mode",
+            "quick",
+            "--output",
+            str(output),
+            "--html-report",
+            str(html),
+        ],
+    )
+
+    assert main() == 0
+    stdout = capsys.readouterr().out
+    assert "FrameVitals Analysis" in stdout
+    assert "Full JSON" in stdout
+    assert "HTML report" in stdout
+
+    payload = json.loads(output.read_text(encoding="utf-8"))
+    assert payload["filename"] == "customers.csv"
+    assert payload["profile"]["shape"]["rows"] == 4
+    assert payload["result_schema_version"] == "1"
+    assert "findings" in payload
+
+    html_text = html.read_text(encoding="utf-8")
+    assert "FrameVitals analysis report" in html_text
+    assert "customers.csv" in html_text
+
+
+def test_analyze_cli_json_stdout_is_complete(tmp_path, monkeypatch, capsys):
+    dataset = tmp_path / "customers.csv"
+    _write_dataset(dataset)
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "analyze",
+            str(dataset),
+            "--mode",
+            "quick",
+            "--format",
+            "json",
+        ],
+    )
+
+    assert main() == 0
+    payload = json.loads(capsys.readouterr().out)
+    assert payload["profile"]["columns"] == ["age", "city"]
+    assert "health" in payload
+    assert "ml_readiness" in payload
+    assert "findings" in payload

From a9932516efaa2ca0d6e8aadc22ca848778376441 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:06:50 +0530
Subject: [PATCH 012/439] feat: add runtime analysis configuration

---
 src/framevitals/config.py | 174 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 174 insertions(+)
 create mode 100644 src/framevitals/config.py

diff --git a/src/framevitals/config.py b/src/framevitals/config.py
new file mode 100644
index 0000000..847dbe5
--- /dev/null
+++ b/src/framevitals/config.py
@@ -0,0 +1,174 @@
+"""Runtime configuration for FrameVitals analysis.
+
+Configuration deliberately starts with options the current pipeline can honor
+fully.  Future category/model/backend settings can extend the same schema
+without introducing a second configuration system.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, replace
+from pathlib import Path
+from typing import Any, Mapping
+import tomllib
+
+
+VALID_MODES = {"quick", "standard", "deep", "research"}
+
+PRESETS: dict[str, dict[str, Any]] = {
+    "quick": {"mode": "quick", "workers": 2, "artifacts": False},
+    "standard": {"mode": "standard", "workers": 4, "artifacts": False},
+    "deep": {"mode": "deep", "workers": 4, "artifacts": False},
+    "research": {"mode": "research", "workers": 4, "artifacts": False},
+    "ci": {"mode": "standard", "workers": 2, "artifacts": False},
+}
+
+
+@dataclass(frozen=True, slots=True)
+class AnalysisConfig:
+    """Resolved configuration consumed by the analysis pipeline."""
+
+    mode: str = "standard"
+    target: str | None = None
+    artifacts: bool = False
+    workers: int = 4
+
+    def __post_init__(self) -> None:
+        if self.mode not in VALID_MODES:
+            raise ValueError(
+                f"Invalid analysis mode '{self.mode}'. "
+                f"Choose from: {', '.join(sorted(VALID_MODES))}"
+            )
+        if self.workers < 1:
+            raise ValueError("workers must be at least 1.")
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+ConfigInput = AnalysisConfig | Mapping[str, Any] | str | Path | None
+
+
+def available_presets() -> tuple[str, ...]:
+    """Return built-in preset names in deterministic order."""
+    return tuple(PRESETS)
+
+
+def _read_toml(path: str | Path) -> dict[str, Any]:
+    source = Path(path)
+    if not source.exists():
+        raise FileNotFoundError(f"FrameVitals config not found: {source}")
+    if not source.is_file():
+        raise ValueError(f"Expected a config file, got: {source}")
+
+    try:
+        with source.open("rb") as handle:
+            payload = tomllib.load(handle)
+    except tomllib.TOMLDecodeError as exc:
+        raise ValueError(f"Invalid TOML in FrameVitals config: {source}") from exc
+
+    if not isinstance(payload, dict):
+        raise ValueError("FrameVitals config must contain a TOML table.")
+    return payload
+
+
+def _extract_values(mapping: Mapping[str, Any]) -> tuple[str | None, dict[str, Any]]:
+    """Extract supported values from nested or flat configuration mappings."""
+    analysis = mapping.get("analysis", {})
+    resources = mapping.get("resources", {})
+
+    if not isinstance(analysis, Mapping):
+        raise ValueError("[analysis] must be a TOML table/object.")
+    if not isinstance(resources, Mapping):
+        raise ValueError("[resources] must be a TOML table/object.")
+
+    preset = analysis.get("preset", mapping.get("preset"))
+    values: dict[str, Any] = {}
+
+    for key in ("mode", "target", "artifacts"):
+        if key in analysis:
+            values[key] = analysis[key]
+        elif key in mapping:
+            values[key] = mapping[key]
+
+    if "workers" in resources:
+        values["workers"] = resources["workers"]
+    elif "workers" in mapping:
+        values["workers"] = mapping["workers"]
+
+    return str(preset) if preset is not None else None, values
+
+
+def _preset_values(name: str | None) -> dict[str, Any]:
+    if name is None:
+        return {}
+    if name not in PRESETS:
+        raise ValueError(
+            f"Unknown FrameVitals preset '{name}'. "
+            f"Choose from: {', '.join(available_presets())}"
+        )
+    return dict(PRESETS[name])
+
+
+def resolve_config(
+    config: ConfigInput = None,
+    *,
+    preset: str | None = None,
+    mode: str | None = None,
+    target: str | None = None,
+    artifacts: bool | None = None,
+    workers: int | None = None,
+) -> AnalysisConfig:
+    """Resolve defaults, preset, config file/object, then explicit overrides.
+
+    Precedence, from lowest to highest, is:
+
+    1. FrameVitals defaults
+    2. explicit ``preset=`` argument
+    3. configuration file/mapping/object
+    4. explicit function/CLI arguments
+    """
+    values: dict[str, Any] = AnalysisConfig().to_dict()
+    values.update(_preset_values(preset))
+
+    if isinstance(config, AnalysisConfig):
+        values.update(config.to_dict())
+    elif isinstance(config, (str, Path)):
+        config_preset, config_values = _extract_values(_read_toml(config))
+        if config_preset is not None:
+            values.update(_preset_values(config_preset))
+        values.update(config_values)
+    elif isinstance(config, Mapping):
+        config_preset, config_values = _extract_values(config)
+        if config_preset is not None:
+            values.update(_preset_values(config_preset))
+        values.update(config_values)
+    elif config is not None:
+        raise TypeError(
+            "config must be an AnalysisConfig, mapping, TOML path, or None."
+        )
+
+    explicit = {
+        "mode": mode,
+        "target": target,
+        "artifacts": artifacts,
+        "workers": workers,
+    }
+    values.update({key: value for key, value in explicit.items() if value is not None})
+
+    try:
+        values["workers"] = int(values["workers"])
+    except (TypeError, ValueError) as exc:
+        raise ValueError("workers must be an integer.") from exc
+
+    if not isinstance(values["artifacts"], bool):
+        raise ValueError("artifacts must be true or false.")
+    if values["target"] is not None and not isinstance(values["target"], str):
+        raise ValueError("target must be a column name string or null.")
+
+    return AnalysisConfig(**values)
+
+
+def with_overrides(config: AnalysisConfig, **changes: Any) -> AnalysisConfig:
+    """Return a validated copy of an existing resolved configuration."""
+    return replace(config, **changes)

From 626fb3672d98c428e2cd5e9d159ea9215722a32f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:07:09 +0530
Subject: [PATCH 013/439] feat: wire config and workers into analyze

---
 src/framevitals/api.py | 81 +++++++++++++++++++++---------------------
 1 file changed, 41 insertions(+), 40 deletions(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index 900e732..991192a 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -7,6 +7,7 @@
 
 import pandas as pd
 
+from framevitals.config import ConfigInput, resolve_config
 from framevitals.contracts import infer_contract as _infer_contract
 from framevitals.contracts import validate_contract
 from framevitals.drift_analysis import compare_datasets
@@ -14,13 +15,6 @@
 from framevitals.pipeline import run_full_analysis
 from framevitals.result import AnalysisResult
 
-VALID_MODES = {
-    "quick",
-    "standard",
-    "deep",
-    "research",
-}
-
 DataInput = str | Path | pd.DataFrame
 
 
@@ -55,8 +49,11 @@ def analyze(
     data: DataInput,
     *,
     target: str | None = None,
-    mode: str = "standard",
-    artifacts: bool = False,
+    mode: str | None = None,
+    artifacts: bool | None = None,
+    workers: int | None = None,
+    preset: str | None = None,
+    config: ConfigInput = None,
 ) -> AnalysisResult:
     """Analyze a tabular dataset with FrameVitals.
 
@@ -65,34 +62,36 @@ def analyze(
     data:
         A pandas DataFrame or a path to a CSV, TSV, Excel, or JSON dataset.
     target:
-        Optional supervised-learning target column.
+        Optional supervised-learning target column. Explicit values override
+        configuration-file values.
     mode:
         Analysis depth: ``quick``, ``standard``, ``deep``, or ``research``.
+        ``None`` resolves to configuration/preset/default behaviour.
     artifacts:
-        When ``True``, persist cleaned CSV/chart artifacts. The reusable Python
-        API defaults to ``False`` so analysis does not modify the filesystem.
+        Persist cleaned CSV/chart artifacts. ``None`` resolves from config and
+        otherwise defaults to ``False`` for reusable library calls.
+    workers:
+        Worker count for independent diagnostic tasks.
+    preset:
+        Optional built-in runtime preset such as ``quick``, ``standard``,
+        ``deep``, ``research``, or ``ci``.
+    config:
+        ``AnalysisConfig``, mapping, or path to a TOML configuration file.
 
     Returns
     -------
     AnalysisResult
         A dict-compatible structured result with summary, column, JSON, HTML,
         and notebook helpers.
-
-    Examples
-    --------
-    >>> import pandas as pd
-    >>> import framevitals as fv
-    >>> df = pd.DataFrame({"age": [20, 30], "income": [30000, 50000]})
-    >>> result = fv.analyze(df, mode="quick")
-    >>> print(result["health"]["overall_score"])
-    >>> print(result.summary())
     """
-    if mode not in VALID_MODES:
-        raise ValueError(
-            f"Invalid analysis mode '{mode}'. "
-            f"Choose from: {', '.join(sorted(VALID_MODES))}"
-        )
-
+    resolved = resolve_config(
+        config,
+        preset=preset,
+        mode=mode,
+        target=target,
+        artifacts=artifacts,
+        workers=workers,
+    )
     dataset_id = f"fv_{uuid4().hex[:12]}"
 
     if isinstance(data, pd.DataFrame):
@@ -102,29 +101,31 @@ def analyze(
             dataset_id=dataset_id,
             dataframe=data,
             original_filename="<dataframe>",
-            analysis_mode=mode,
-            target_column=target,
+            analysis_mode=resolved.mode,
+            target_column=resolved.target,
+            parallel_workers=resolved.workers,
             skip_ai=True,
-            write_artifacts=artifacts,
+            write_artifacts=resolved.artifacts,
         )
-        return AnalysisResult(payload)
-
-    if isinstance(data, (str, Path)):
+    elif isinstance(data, (str, Path)):
         path = _validated_path(data)
         payload = run_full_analysis(
             dataset_id=dataset_id,
             file_path=path,
             original_filename=path.name,
-            analysis_mode=mode,
-            target_column=target,
+            analysis_mode=resolved.mode,
+            target_column=resolved.target,
+            parallel_workers=resolved.workers,
             skip_ai=True,
-            write_artifacts=artifacts,
+            write_artifacts=resolved.artifacts,
+        )
+    else:
+        raise TypeError(
+            "data must be a pandas DataFrame or a path to a supported dataset."
         )
-        return AnalysisResult(payload)
 
-    raise TypeError(
-        "data must be a pandas DataFrame or a path to a supported dataset."
-    )
+    payload["config"] = resolved.to_dict()
+    return AnalysisResult(payload)
 
 
 def compare(

From 8eb4b4d0fb696285754ab11a256398f99856d2df Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:07:22 +0530
Subject: [PATCH 014/439] feat: expose AnalysisConfig in public API

---
 src/framevitals/__init__.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index fdeb0b4..95f4ded 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -4,6 +4,7 @@
 
 from typing import Any
 
+from framevitals.config import AnalysisConfig
 from framevitals.result import AnalysisResult, ColumnResult
 
 __version__ = "0.1.0"
@@ -13,14 +14,17 @@ def analyze(
     data: Any,
     *,
     target: str | None = None,
-    mode: str = "standard",
-    artifacts: bool = False,
+    mode: str | None = None,
+    artifacts: bool | None = None,
+    workers: int | None = None,
+    preset: str | None = None,
+    config: Any = None,
 ) -> AnalysisResult:
     """Analyze a tabular dataset from a DataFrame or supported file path.
 
-    The analytics implementation is imported lazily so ``import framevitals``
-    and ``framevitals --version`` do not initialize the complete analytics
-    stack. The lightweight result types remain available at package level.
+    Explicit keyword arguments override values from ``config`` and ``preset``.
+    The analytics implementation is imported lazily so importing FrameVitals
+    does not initialize the complete analysis stack.
     """
     from framevitals.api import analyze as _analyze
 
@@ -29,6 +33,9 @@ def analyze(
         target=target,
         mode=mode,
         artifacts=artifacts,
+        workers=workers,
+        preset=preset,
+        config=config,
     )
 
 
@@ -65,6 +72,7 @@ def validate(data: Any, contract: dict[str, Any]) -> dict[str, Any]:
 
 
 __all__ = [
+    "AnalysisConfig",
     "AnalysisResult",
     "ColumnResult",
     "analyze",

From 5c91ba6fbd9df2dc6b43c2e02c371ebebf16a741 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:07:42 +0530
Subject: [PATCH 015/439] feat: add CLI config and preset controls

---
 src/framevitals/cli.py | 65 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 60 insertions(+), 5 deletions(-)

diff --git a/src/framevitals/cli.py b/src/framevitals/cli.py
index 0b24008..26d8198 100644
--- a/src/framevitals/cli.py
+++ b/src/framevitals/cli.py
@@ -3,6 +3,7 @@
 from pathlib import Path
 
 from framevitals import __version__
+from framevitals.config import available_presets
 
 
 def _add_output_argument(parser: argparse.ArgumentParser) -> None:
@@ -56,18 +57,37 @@ def build_parser() -> argparse.ArgumentParser:
     analyze_parser.add_argument(
         "--target",
         default=None,
-        help="Optional target column.",
+        help="Optional target column. Overrides config values.",
     )
     analyze_parser.add_argument(
         "--mode",
         choices=["quick", "standard", "deep", "research"],
-        default="standard",
-        help="Analysis depth.",
+        default=None,
+        help="Analysis depth. Overrides preset/config values.",
+    )
+    analyze_parser.add_argument(
+        "--preset",
+        choices=list(available_presets()),
+        default=None,
+        help="Built-in runtime preset.",
+    )
+    analyze_parser.add_argument(
+        "--config",
+        type=Path,
+        default=None,
+        help="Optional FrameVitals TOML config path.",
+    )
+    analyze_parser.add_argument(
+        "--workers",
+        type=int,
+        default=None,
+        help="Parallel worker count. Overrides config values.",
     )
     analyze_parser.add_argument(
         "--artifacts",
-        action="store_true",
-        help="Persist cleaned CSV/chart artifacts.",
+        action=argparse.BooleanOptionalAction,
+        default=None,
+        help="Enable/disable cleaned CSV and chart artifacts.",
     )
     analyze_parser.add_argument(
         "--format",
@@ -138,6 +158,23 @@ def build_parser() -> argparse.ArgumentParser:
     )
     _add_output_argument(validate_parser)
 
+    config_parser = subparsers.add_parser(
+        "config",
+        help="Resolve and inspect FrameVitals runtime configuration.",
+    )
+    config_parser.add_argument(
+        "--file",
+        type=Path,
+        default=None,
+        help="Optional TOML config file.",
+    )
+    config_parser.add_argument(
+        "--preset",
+        choices=list(available_presets()),
+        default=None,
+        help="Optional built-in preset to resolve before the config file.",
+    )
+
     return parser
 
 
@@ -161,6 +198,9 @@ def main() -> int:
             target=args.target,
             mode=args.mode,
             artifacts=args.artifacts,
+            workers=args.workers,
+            preset=args.preset,
+            config=args.config,
         )
 
         if args.output is not None:
@@ -172,6 +212,14 @@ def main() -> int:
             print(report.to_json())
         else:
             print(report.summary_text())
+            resolved = report.get("config", {})
+            if resolved:
+                print(
+                    "Config        "
+                    f"mode={resolved.get('mode')} "
+                    f"workers={resolved.get('workers')} "
+                    f"artifacts={resolved.get('artifacts')}"
+                )
             if args.output is not None:
                 print(f"Full JSON     {args.output}")
             if args.html_report is not None:
@@ -215,6 +263,13 @@ def main() -> int:
         _emit(report, args.output)
         return 0 if report["valid"] else 1
 
+    if args.command == "config":
+        from framevitals.config import resolve_config
+
+        resolved = resolve_config(args.file, preset=args.preset)
+        print(json.dumps(resolved.to_dict(), indent=2))
+        return 0
+
     parser.print_help()
     return 0
 

From ae0da995871ba0a6d0b14204f9d86340f6c88fb2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:07:58 +0530
Subject: [PATCH 016/439] test: update CLI parser coverage for config controls

---
 tests/test_cli.py | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/tests/test_cli.py b/tests/test_cli.py
index f33839f..9bedb0b 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -17,7 +17,10 @@ def test_cli_parser():
     assert args.file.name == "dataset.csv"
     assert args.mode == "quick"
     assert args.target is None
-    assert args.artifacts is False
+    assert args.artifacts is None
+    assert args.workers is None
+    assert args.preset is None
+    assert args.config is None
     assert args.output is None
 
 
@@ -32,6 +35,8 @@ def test_cli_target_argument():
         "--mode",
         "deep",
         "--artifacts",
+        "--workers",
+        "6",
         "--output",
         "result.json",
     ])
@@ -40,9 +45,38 @@ def test_cli_target_argument():
     assert args.target == "churn"
     assert args.mode == "deep"
     assert args.artifacts is True
+    assert args.workers == 6
     assert args.output.name == "result.json"
 
 
+def test_cli_config_and_preset_arguments():
+    parser = build_parser()
+
+    args = parser.parse_args([
+        "analyze",
+        "customers.csv",
+        "--preset",
+        "ci",
+        "--config",
+        "framevitals.toml",
+        "--no-artifacts",
+    ])
+    config_args = parser.parse_args([
+        "config",
+        "--file",
+        "framevitals.toml",
+        "--preset",
+        "deep",
+    ])
+
+    assert args.preset == "ci"
+    assert args.config.name == "framevitals.toml"
+    assert args.artifacts is False
+    assert config_args.command == "config"
+    assert config_args.file.name == "framevitals.toml"
+    assert config_args.preset == "deep"
+
+
 def test_cli_compare_parser():
     parser = build_parser()
 

From a1c2bdd8a578edff7c393b71d9d9d6c625d94c59 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:08:09 +0530
Subject: [PATCH 017/439] test: cover TOML config and precedence

---
 tests/test_config.py | 88 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 88 insertions(+)
 create mode 100644 tests/test_config.py

diff --git a/tests/test_config.py b/tests/test_config.py
new file mode 100644
index 0000000..2fd1a2d
--- /dev/null
+++ b/tests/test_config.py
@@ -0,0 +1,88 @@
+import pandas as pd
+import pytest
+
+import framevitals
+from framevitals.config import AnalysisConfig, resolve_config
+
+
+def test_default_and_preset_resolution():
+    assert resolve_config() == AnalysisConfig()
+
+    quick = resolve_config(preset="quick")
+    assert quick.mode == "quick"
+    assert quick.workers == 2
+    assert quick.artifacts is False
+
+
+def test_toml_config_resolution_and_explicit_precedence(tmp_path):
+    config = tmp_path / "framevitals.toml"
+    config.write_text(
+        "[analysis]\n"
+        "preset = \"deep\"\n"
+        "target = \"churn\"\n"
+        "artifacts = true\n"
+        "\n"
+        "[resources]\n"
+        "workers = 6\n",
+        encoding="utf-8",
+    )
+
+    resolved = resolve_config(config)
+    assert resolved.mode == "deep"
+    assert resolved.target == "churn"
+    assert resolved.artifacts is True
+    assert resolved.workers == 6
+
+    overridden = resolve_config(
+        config,
+        mode="quick",
+        target="label",
+        artifacts=False,
+        workers=3,
+    )
+    assert overridden == AnalysisConfig(
+        mode="quick",
+        target="label",
+        artifacts=False,
+        workers=3,
+    )
+
+
+def test_config_mapping_and_validation():
+    resolved = resolve_config({
+        "analysis": {"mode": "research", "artifacts": False},
+        "resources": {"workers": 8},
+    })
+    assert resolved.mode == "research"
+    assert resolved.workers == 8
+
+    with pytest.raises(ValueError, match="Unknown FrameVitals preset"):
+        resolve_config(preset="everything")
+    with pytest.raises(ValueError, match="workers"):
+        resolve_config(workers=0)
+
+
+def test_public_analyze_honors_config_and_records_resolution(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    df = pd.DataFrame({
+        "age": [20, 30, 40, 50],
+        "city": ["Pune", "Mumbai", "Pune", "Nashik"],
+    })
+
+    result = framevitals.analyze(
+        df,
+        config={
+            "analysis": {"mode": "quick", "artifacts": False},
+            "resources": {"workers": 1},
+        },
+    )
+
+    assert result["analysis_mode"] == "quick"
+    assert result["artifacts_enabled"] is False
+    assert result["config"] == {
+        "mode": "quick",
+        "target": None,
+        "artifacts": False,
+        "workers": 1,
+    }
+    assert not (tmp_path / "cleaned").exists()

From 4ef8d660a79f512f0fb9e3441de4bbe41ebde542 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:08:46 +0530
Subject: [PATCH 018/439] feat: add AnalysisPlan result type

---
 src/framevitals/planning.py | 88 +++++++++++++++++++++++++++++++++++++
 1 file changed, 88 insertions(+)
 create mode 100644 src/framevitals/planning.py

diff --git a/src/framevitals/planning.py b/src/framevitals/planning.py
new file mode 100644
index 0000000..43d0e61
--- /dev/null
+++ b/src/framevitals/planning.py
@@ -0,0 +1,88 @@
+"""Analysis planning result objects and human-readable explanations."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class AnalysisPlan(dict):
+    """Dict-compatible preview of analyses FrameVitals considers applicable."""
+
+    def __getattr__(self, name: str) -> Any:
+        try:
+            return self[name]
+        except KeyError as exc:
+            raise AttributeError(name) from exc
+
+    @property
+    def selection(self) -> dict[str, Any]:
+        value = self.get("selection", {})
+        return value if isinstance(value, dict) else {}
+
+    @property
+    def selected(self) -> list[dict[str, Any]]:
+        value = self.selection.get("selected_analyses", [])
+        return value if isinstance(value, list) else []
+
+    @property
+    def skipped(self) -> list[dict[str, Any]]:
+        value = self.selection.get("skipped_analyses", [])
+        return value if isinstance(value, list) else []
+
+    @property
+    def recommended(self) -> list[dict[str, Any]]:
+        value = self.selection.get("recommended_analyses", [])
+        return value if isinstance(value, list) else []
+
+    def summary(self) -> dict[str, Any]:
+        return {
+            "dataset_name": self.get("dataset_name"),
+            "analysis_mode": self.get("analysis_mode"),
+            "target": self.get("target"),
+            "shape": self.get("shape", {}),
+            "selected_count": len(self.selected),
+            "skipped_count": len(self.skipped),
+            "recommended_count": len(self.recommended),
+        }
+
+    def explain_text(self) -> str:
+        """Render a terminal-friendly explanation of the current plan."""
+        shape = self.get("shape", {}) or {}
+        lines = [
+            "FrameVitals Analysis Plan",
+            "=" * 72,
+            f"Dataset       {self.get('dataset_name', '<unknown>')}",
+            f"Mode          {self.get('analysis_mode', 'unknown')}",
+            f"Target        {self.get('target') or '<none>'}",
+            f"Shape         {shape.get('rows', '?')} rows x {shape.get('columns', '?')} columns",
+            "",
+            f"Selected      {len(self.selected)}",
+        ]
+
+        for item in self.selected:
+            lines.append(
+                f"  [RUN]  {item.get('id', ''):<28} {item.get('name', '')}"
+            )
+
+        lines.extend(["", f"Recommended   {len(self.recommended)}"])
+        for item in self.recommended:
+            lines.append(
+                f"  [NEXT] {item.get('id', ''):<28} {item.get('reason', '')}"
+            )
+
+        lines.extend(["", f"Skipped       {len(self.skipped)}"])
+        for item in self.skipped[:12]:
+            reason = " ".join(str(item.get("reason", "")).split())
+            if len(reason) > 68:
+                reason = reason[:67].rstrip() + "…"
+            lines.append(
+                f"  [SKIP] {item.get('id', ''):<28} {reason}"
+            )
+        if len(self.skipped) > 12:
+            lines.append(f"  ... and {len(self.skipped) - 12} more skipped analyses")
+
+        lines.extend([
+            "=" * 72,
+            "This is a preview only; no heavy model/statistics stage was executed.",
+        ])
+        return "\n".join(lines)

From 643ae5c0f1a8a31532f3c3d6933939d64c4a9881 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:09:06 +0530
Subject: [PATCH 019/439] feat: add public analysis planning API

---
 src/framevitals/api.py | 109 +++++++++++++++++++++++------------------
 1 file changed, 62 insertions(+), 47 deletions(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index 991192a..e789dc5 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -7,12 +7,17 @@
 
 import pandas as pd
 
+from framevitals.analysis_selector import select_analyses
+from framevitals.column_roles import infer_column_roles
 from framevitals.config import ConfigInput, resolve_config
 from framevitals.contracts import infer_contract as _infer_contract
 from framevitals.contracts import validate_contract
+from framevitals.dataset_signals import detect_dataset_signals
 from framevitals.drift_analysis import compare_datasets
 from framevitals.loader import load_dataset
 from framevitals.pipeline import run_full_analysis
+from framevitals.planning import AnalysisPlan
+from framevitals.profiler import build_profile
 from framevitals.result import AnalysisResult
 
 DataInput = str | Path | pd.DataFrame
@@ -55,35 +60,7 @@ def analyze(
     preset: str | None = None,
     config: ConfigInput = None,
 ) -> AnalysisResult:
-    """Analyze a tabular dataset with FrameVitals.
-
-    Parameters
-    ----------
-    data:
-        A pandas DataFrame or a path to a CSV, TSV, Excel, or JSON dataset.
-    target:
-        Optional supervised-learning target column. Explicit values override
-        configuration-file values.
-    mode:
-        Analysis depth: ``quick``, ``standard``, ``deep``, or ``research``.
-        ``None`` resolves to configuration/preset/default behaviour.
-    artifacts:
-        Persist cleaned CSV/chart artifacts. ``None`` resolves from config and
-        otherwise defaults to ``False`` for reusable library calls.
-    workers:
-        Worker count for independent diagnostic tasks.
-    preset:
-        Optional built-in runtime preset such as ``quick``, ``standard``,
-        ``deep``, ``research``, or ``ci``.
-    config:
-        ``AnalysisConfig``, mapping, or path to a TOML configuration file.
-
-    Returns
-    -------
-    AnalysisResult
-        A dict-compatible structured result with summary, column, JSON, HTML,
-        and notebook helpers.
-    """
+    """Analyze a tabular dataset with FrameVitals."""
     resolved = resolve_config(
         config,
         preset=preset,
@@ -128,6 +105,59 @@ def analyze(
     return AnalysisResult(payload)
 
 
+def plan(
+    data: DataInput,
+    *,
+    target: str | None = None,
+    mode: str | None = None,
+    workers: int | None = None,
+    preset: str | None = None,
+    config: ConfigInput = None,
+) -> AnalysisPlan:
+    """Preview which analyses FrameVitals considers applicable.
+
+    Planning performs loading, profiling, role inference, signal detection, and
+    selector evaluation only. It does not run the heavier model/statistics,
+    cleaning, chart, or AI stages of :func:`analyze`.
+    """
+    resolved = resolve_config(
+        config,
+        preset=preset,
+        mode=mode,
+        target=target,
+        workers=workers,
+        artifacts=False,
+    )
+    dataframe, source_name = _load_input(data, label="Dataset")
+    profile = build_profile(dataframe)
+    column_roles = infer_column_roles(dataframe)
+    dataset_signals = detect_dataset_signals(
+        dataframe,
+        profile,
+        column_roles=column_roles,
+    )
+    selection = select_analyses(
+        signals=dataset_signals,
+        analysis_mode=resolved.mode,
+        target_column=resolved.target,
+    )
+
+    public_signals = {
+        key: value
+        for key, value in dataset_signals.items()
+        if key != "column_roles"
+    }
+    return AnalysisPlan({
+        "dataset_name": source_name,
+        "analysis_mode": resolved.mode,
+        "target": resolved.target,
+        "shape": dict(profile.get("shape", {})),
+        "config": resolved.to_dict(),
+        "signals": public_signals,
+        "selection": selection,
+    })
+
+
 def compare(
     reference: DataInput,
     current: DataInput,
@@ -135,12 +165,7 @@ def compare(
     columns: list[str] | None = None,
     max_columns: int = 30,
 ) -> dict:
-    """Compare reference and current datasets for distribution drift.
-
-    Both inputs may independently be pandas DataFrames or supported dataset
-    paths. Numeric features use PSI, KS statistics, and standardized mean
-    shift; categorical features use PSI and chi-square diagnostics.
-    """
+    """Compare reference and current datasets for distribution drift."""
     if max_columns < 1:
         raise ValueError("max_columns must be at least 1.")
 
@@ -159,11 +184,7 @@ def compare(
 
 
 def infer_contract(data: DataInput) -> dict[str, Any]:
-    """Infer a JSON-serializable data contract from a reference dataset.
-
-    The result can be saved with :mod:`json` and passed to :func:`validate`
-    when checking later datasets in a pipeline or CI job.
-    """
+    """Infer a JSON-serializable data contract from a reference dataset."""
     dataframe, source_name = _load_input(data, label="Reference")
     contract = _infer_contract(dataframe)
     contract["reference_name"] = source_name
@@ -174,13 +195,7 @@ def validate(
     data: DataInput,
     contract: Mapping[str, Any],
 ) -> dict[str, Any]:
-    """Validate a dataset against an inferred or explicit data contract.
-
-    Contract failures are returned as structured findings rather than raised,
-    allowing callers to decide whether warnings or errors should block a job.
-    Invalid contract definitions and unreadable datasets still raise clear
-    exceptions.
-    """
+    """Validate a dataset against an inferred or explicit data contract."""
     dataframe, source_name = _load_input(data, label="Dataset")
     result = validate_contract(dataframe, contract)
     result["dataset_name"] = source_name

From da910aed0212467d9a5882dfac8d2bc79189c85f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:09:19 +0530
Subject: [PATCH 020/439] feat: expose public analysis plan API

---
 src/framevitals/__init__.py | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 95f4ded..2061d48 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -5,6 +5,7 @@
 from typing import Any
 
 from framevitals.config import AnalysisConfig
+from framevitals.planning import AnalysisPlan
 from framevitals.result import AnalysisResult, ColumnResult
 
 __version__ = "0.1.0"
@@ -20,12 +21,7 @@ def analyze(
     preset: str | None = None,
     config: Any = None,
 ) -> AnalysisResult:
-    """Analyze a tabular dataset from a DataFrame or supported file path.
-
-    Explicit keyword arguments override values from ``config`` and ``preset``.
-    The analytics implementation is imported lazily so importing FrameVitals
-    does not initialize the complete analysis stack.
-    """
+    """Analyze a tabular dataset from a DataFrame or supported file path."""
     from framevitals.api import analyze as _analyze
 
     return _analyze(
@@ -39,6 +35,28 @@ def analyze(
     )
 
 
+def plan(
+    data: Any,
+    *,
+    target: str | None = None,
+    mode: str | None = None,
+    workers: int | None = None,
+    preset: str | None = None,
+    config: Any = None,
+) -> AnalysisPlan:
+    """Preview applicable analyses without running heavy analysis stages."""
+    from framevitals.api import plan as _plan
+
+    return _plan(
+        data,
+        target=target,
+        mode=mode,
+        workers=workers,
+        preset=preset,
+        config=config,
+    )
+
+
 def compare(
     reference: Any,
     current: Any,
@@ -73,9 +91,11 @@ def validate(data: Any, contract: dict[str, Any]) -> dict[str, Any]:
 
 __all__ = [
     "AnalysisConfig",
+    "AnalysisPlan",
     "AnalysisResult",
     "ColumnResult",
     "analyze",
+    "plan",
     "compare",
     "infer_contract",
     "validate",

From 93f998f2a245c69f37f1829f4208ef041311941c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:09:42 +0530
Subject: [PATCH 021/439] feat: add analysis plan CLI command

---
 src/framevitals/cli.py | 105 +++++++++++++++++++++++++++++------------
 1 file changed, 76 insertions(+), 29 deletions(-)

diff --git a/src/framevitals/cli.py b/src/framevitals/cli.py
index 26d8198..fc62b2f 100644
--- a/src/framevitals/cli.py
+++ b/src/framevitals/cli.py
@@ -15,6 +15,38 @@ def _add_output_argument(parser: argparse.ArgumentParser) -> None:
     )
 
 
+def _add_runtime_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--target",
+        default=None,
+        help="Optional target column. Overrides config values.",
+    )
+    parser.add_argument(
+        "--mode",
+        choices=["quick", "standard", "deep", "research"],
+        default=None,
+        help="Analysis depth. Overrides preset/config values.",
+    )
+    parser.add_argument(
+        "--preset",
+        choices=list(available_presets()),
+        default=None,
+        help="Built-in runtime preset.",
+    )
+    parser.add_argument(
+        "--config",
+        type=Path,
+        default=None,
+        help="Optional FrameVitals TOML config path.",
+    )
+    parser.add_argument(
+        "--workers",
+        type=int,
+        default=None,
+        help="Parallel worker count. Overrides config values.",
+    )
+
+
 def _load_contract(path: Path) -> dict:
     try:
         payload = json.loads(path.read_text(encoding="utf-8"))
@@ -54,35 +86,7 @@ def build_parser() -> argparse.ArgumentParser:
         type=Path,
         help="Path to the dataset.",
     )
-    analyze_parser.add_argument(
-        "--target",
-        default=None,
-        help="Optional target column. Overrides config values.",
-    )
-    analyze_parser.add_argument(
-        "--mode",
-        choices=["quick", "standard", "deep", "research"],
-        default=None,
-        help="Analysis depth. Overrides preset/config values.",
-    )
-    analyze_parser.add_argument(
-        "--preset",
-        choices=list(available_presets()),
-        default=None,
-        help="Built-in runtime preset.",
-    )
-    analyze_parser.add_argument(
-        "--config",
-        type=Path,
-        default=None,
-        help="Optional FrameVitals TOML config path.",
-    )
-    analyze_parser.add_argument(
-        "--workers",
-        type=int,
-        default=None,
-        help="Parallel worker count. Overrides config values.",
-    )
+    _add_runtime_arguments(analyze_parser)
     analyze_parser.add_argument(
         "--artifacts",
         action=argparse.BooleanOptionalAction,
@@ -103,6 +107,24 @@ def build_parser() -> argparse.ArgumentParser:
     )
     _add_output_argument(analyze_parser)
 
+    plan_parser = subparsers.add_parser(
+        "plan",
+        help="Preview applicable analyses without running heavy stages.",
+    )
+    plan_parser.add_argument(
+        "file",
+        type=Path,
+        help="Path to the dataset.",
+    )
+    _add_runtime_arguments(plan_parser)
+    plan_parser.add_argument(
+        "--format",
+        choices=["terminal", "json"],
+        default="terminal",
+        help="Plan output format.",
+    )
+    _add_output_argument(plan_parser)
+
     compare_parser = subparsers.add_parser(
         "compare",
         help="Compare reference and current datasets for drift.",
@@ -226,6 +248,31 @@ def main() -> int:
                 print(f"HTML report   {args.html_report}")
         return 0
 
+    if args.command == "plan":
+        from framevitals.api import plan
+
+        result = plan(
+            args.file,
+            target=args.target,
+            mode=args.mode,
+            workers=args.workers,
+            preset=args.preset,
+            config=args.config,
+        )
+        if args.output is not None:
+            args.output.parent.mkdir(parents=True, exist_ok=True)
+            args.output.write_text(
+                json.dumps(dict(result), indent=2, default=str) + "\n",
+                encoding="utf-8",
+            )
+        if args.format == "json":
+            print(json.dumps(dict(result), indent=2, default=str))
+        else:
+            print(result.explain_text())
+            if args.output is not None:
+                print(f"Plan JSON     {args.output}")
+        return 0
+
     if args.command == "compare":
         from framevitals.api import compare
 

From 28030e46ae024a3814bbbdad9d7a33c228cd8f11 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:09:55 +0530
Subject: [PATCH 022/439] test: cover public and CLI analysis planning

---
 tests/test_planning.py | 86 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 tests/test_planning.py

diff --git a/tests/test_planning.py b/tests/test_planning.py
new file mode 100644
index 0000000..e9bfa59
--- /dev/null
+++ b/tests/test_planning.py
@@ -0,0 +1,86 @@
+import json
+
+import pandas as pd
+
+import framevitals
+from framevitals.cli import main
+
+
+def _dataset() -> pd.DataFrame:
+    return pd.DataFrame({
+        "age": list(range(20, 40)),
+        "income": [30000 + index * 1000 for index in range(20)],
+        "city": ["Pune", "Mumbai"] * 10,
+        "churn": [0, 1] * 10,
+    })
+
+
+def test_public_plan_returns_explainable_dict_compatible_plan():
+    result = framevitals.plan(_dataset(), mode="quick")
+
+    assert isinstance(result, framevitals.AnalysisPlan)
+    assert isinstance(result, dict)
+    assert result["dataset_name"] == "<dataframe>"
+    assert result.summary()["selected_count"] > 0
+    assert any(item["id"] == "structural_profile" for item in result.selected)
+    assert "FrameVitals Analysis Plan" in result.explain_text()
+    assert "preview only" in result.explain_text().lower()
+
+
+def test_deep_plan_with_target_selects_target_aware_work():
+    result = framevitals.plan(_dataset(), mode="deep", target="churn")
+    selected_ids = {item["id"] for item in result.selected}
+
+    assert "target_analysis" in selected_ids
+    assert "baseline_model" in selected_ids
+    assert "feature_importance" in selected_ids
+    assert result["config"]["target"] == "churn"
+
+
+def test_plan_uses_config_without_writing_artifacts(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    config = tmp_path / "framevitals.toml"
+    config.write_text(
+        "[analysis]\nmode = \"deep\"\ntarget = \"churn\"\nartifacts = true\n"
+        "[resources]\nworkers = 1\n",
+        encoding="utf-8",
+    )
+
+    result = framevitals.plan(_dataset(), config=config)
+
+    assert result["analysis_mode"] == "deep"
+    assert result["target"] == "churn"
+    assert result["config"]["artifacts"] is False
+    assert not (tmp_path / "cleaned").exists()
+    assert not (tmp_path / "static").exists()
+
+
+def test_cli_plan_can_emit_json(tmp_path, monkeypatch, capsys):
+    dataset = tmp_path / "dataset.csv"
+    _dataset().to_csv(dataset, index=False)
+    output = tmp_path / "plan.json"
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "plan",
+            str(dataset),
+            "--mode",
+            "deep",
+            "--target",
+            "churn",
+            "--format",
+            "json",
+            "--output",
+            str(output),
+        ],
+    )
+
+    assert main() == 0
+    stdout_payload = json.loads(capsys.readouterr().out)
+    file_payload = json.loads(output.read_text(encoding="utf-8"))
+
+    assert stdout_payload["analysis_mode"] == "deep"
+    assert file_payload["target"] == "churn"
+    assert file_payload["selection"]["summary"]["selected_count"] > 0

From 54b34e4ab2e42f372cd5eeb1936a1cc95e8cf26a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:10:49 +0530
Subject: [PATCH 023/439] feat: add compact analysis snapshots and diffs

---
 src/framevitals/snapshots.py | 209 +++++++++++++++++++++++++++++++++++
 1 file changed, 209 insertions(+)
 create mode 100644 src/framevitals/snapshots.py

diff --git a/src/framevitals/snapshots.py b/src/framevitals/snapshots.py
new file mode 100644
index 0000000..8892919
--- /dev/null
+++ b/src/framevitals/snapshots.py
@@ -0,0 +1,209 @@
+"""Compact versioned snapshots derived from FrameVitals analysis results."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Mapping
+
+
+SNAPSHOT_SCHEMA_VERSION = "1"
+
+
+def _as_mapping(value: Any) -> Mapping[str, Any]:
+    return value if isinstance(value, Mapping) else {}
+
+
+def _number(value: Any) -> float | None:
+    try:
+        if value is None:
+            return None
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _state_payload(result: Mapping[str, Any]) -> dict[str, Any]:
+    profile = _as_mapping(result.get("profile"))
+    health = _as_mapping(result.get("health"))
+    ml = _as_mapping(result.get("ml_readiness"))
+    findings = result.get("findings", [])
+    if not isinstance(findings, list):
+        findings = []
+
+    return {
+        "result_schema_version": result.get("result_schema_version"),
+        "analysis_mode": result.get("analysis_mode"),
+        "dataset": {
+            "shape": dict(_as_mapping(profile.get("shape"))),
+            "dtypes": dict(_as_mapping(profile.get("dtypes"))),
+            "missing_percent": dict(_as_mapping(profile.get("missing_percent"))),
+            "duplicate_percent": profile.get("duplicate_percent"),
+            "memory_usage_mb": profile.get("memory_usage_mb"),
+        },
+        "health": {
+            "overall_score": health.get("overall_score"),
+            "label": health.get("label"),
+            "components": dict(_as_mapping(health.get("components"))),
+        },
+        "ml_readiness": {
+            "score": ml.get("score"),
+            "label": ml.get("label"),
+            "issues": dict(_as_mapping(ml.get("issues"))),
+        },
+        "finding_codes": sorted(
+            str(item.get("code"))
+            for item in findings
+            if isinstance(item, Mapping) and item.get("code")
+        ),
+        "config": dict(_as_mapping(result.get("config"))),
+    }
+
+
+def _fingerprint(state: Mapping[str, Any]) -> str:
+    canonical = json.dumps(
+        dict(state),
+        sort_keys=True,
+        separators=(",", ":"),
+        default=str,
+    )
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+class AnalysisSnapshot(dict):
+    """Small JSON-friendly state record for monitoring and history."""
+
+    def __getattr__(self, name: str) -> Any:
+        try:
+            return self[name]
+        except KeyError as exc:
+            raise AttributeError(name) from exc
+
+    def to_json(
+        self,
+        destination: str | Path | None = None,
+        *,
+        indent: int = 2,
+    ) -> str | Path:
+        rendered = json.dumps(dict(self), indent=indent, default=str)
+        if destination is None:
+            return rendered
+        path = Path(destination)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(rendered + "\n", encoding="utf-8")
+        return path
+
+    def diff(self, other: Mapping[str, Any]) -> dict[str, Any]:
+        return compare_snapshots(self, other)
+
+
+def create_snapshot(result: Mapping[str, Any]) -> AnalysisSnapshot:
+    """Create a deterministic compact state snapshot from an analysis result."""
+    state = _state_payload(result)
+    return AnalysisSnapshot({
+        "snapshot_schema_version": SNAPSHOT_SCHEMA_VERSION,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "source": {
+            "dataset_id": result.get("dataset_id"),
+            "filename": result.get("filename"),
+        },
+        "fingerprint": _fingerprint(state),
+        "state": state,
+    })
+
+
+def load_snapshot(path: str | Path) -> AnalysisSnapshot:
+    """Load and validate a FrameVitals snapshot from JSON."""
+    source = Path(path)
+    if not source.exists():
+        raise FileNotFoundError(f"Snapshot not found: {source}")
+    try:
+        payload = json.loads(source.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Snapshot is not valid JSON: {source}") from exc
+
+    if not isinstance(payload, dict):
+        raise ValueError("Snapshot JSON must contain an object.")
+    if payload.get("snapshot_schema_version") != SNAPSHOT_SCHEMA_VERSION:
+        raise ValueError(
+            "Unsupported FrameVitals snapshot schema version: "
+            f"{payload.get('snapshot_schema_version')!r}"
+        )
+    if not isinstance(payload.get("state"), dict):
+        raise ValueError("Snapshot is missing a valid state object.")
+    return AnalysisSnapshot(payload)
+
+
+def compare_snapshots(
+    reference: Mapping[str, Any],
+    current: Mapping[str, Any],
+) -> dict[str, Any]:
+    """Compare compact analysis snapshots without requiring raw datasets."""
+    ref_state = _as_mapping(reference.get("state"))
+    cur_state = _as_mapping(current.get("state"))
+    ref_dataset = _as_mapping(ref_state.get("dataset"))
+    cur_dataset = _as_mapping(cur_state.get("dataset"))
+
+    ref_dtypes = dict(_as_mapping(ref_dataset.get("dtypes")))
+    cur_dtypes = dict(_as_mapping(cur_dataset.get("dtypes")))
+    ref_columns = set(ref_dtypes)
+    cur_columns = set(cur_dtypes)
+
+    type_changes = {
+        column: {"reference": ref_dtypes[column], "current": cur_dtypes[column]}
+        for column in sorted(ref_columns & cur_columns)
+        if ref_dtypes[column] != cur_dtypes[column]
+    }
+
+    ref_missing = dict(_as_mapping(ref_dataset.get("missing_percent")))
+    cur_missing = dict(_as_mapping(cur_dataset.get("missing_percent")))
+    missing_changes: dict[str, dict[str, float]] = {}
+    for column in sorted(set(ref_missing) & set(cur_missing)):
+        before = _number(ref_missing[column])
+        after = _number(cur_missing[column])
+        if before is None or after is None or before == after:
+            continue
+        missing_changes[column] = {
+            "reference": round(before, 4),
+            "current": round(after, 4),
+            "delta": round(after - before, 4),
+        }
+
+    ref_health = _number(_as_mapping(ref_state.get("health")).get("overall_score"))
+    cur_health = _number(_as_mapping(cur_state.get("health")).get("overall_score"))
+    ref_ml = _number(_as_mapping(ref_state.get("ml_readiness")).get("score"))
+    cur_ml = _number(_as_mapping(cur_state.get("ml_readiness")).get("score"))
+
+    ref_findings = set(ref_state.get("finding_codes", []) or [])
+    cur_findings = set(cur_state.get("finding_codes", []) or [])
+
+    changed = bool(
+        reference.get("fingerprint") != current.get("fingerprint")
+    )
+    return {
+        "changed": changed,
+        "reference_fingerprint": reference.get("fingerprint"),
+        "current_fingerprint": current.get("fingerprint"),
+        "schema": {
+            "added_columns": sorted(cur_columns - ref_columns),
+            "removed_columns": sorted(ref_columns - cur_columns),
+            "type_changes": type_changes,
+        },
+        "missingness_changes": missing_changes,
+        "health_delta": (
+            round(cur_health - ref_health, 4)
+            if ref_health is not None and cur_health is not None
+            else None
+        ),
+        "ml_readiness_delta": (
+            round(cur_ml - ref_ml, 4)
+            if ref_ml is not None and cur_ml is not None
+            else None
+        ),
+        "findings": {
+            "new": sorted(cur_findings - ref_findings),
+            "resolved": sorted(ref_findings - cur_findings),
+        },
+    }

From 729500baf92284657274a68955ba4752f1164f54 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:11:07 +0530
Subject: [PATCH 024/439] feat: expose snapshot monitoring primitives

---
 src/framevitals/__init__.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 2061d48..b3bd591 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -7,6 +7,12 @@
 from framevitals.config import AnalysisConfig
 from framevitals.planning import AnalysisPlan
 from framevitals.result import AnalysisResult, ColumnResult
+from framevitals.snapshots import (
+    AnalysisSnapshot,
+    compare_snapshots,
+    create_snapshot,
+    load_snapshot,
+)
 
 __version__ = "0.1.0"
 
@@ -93,11 +99,15 @@ def validate(data: Any, contract: dict[str, Any]) -> dict[str, Any]:
     "AnalysisConfig",
     "AnalysisPlan",
     "AnalysisResult",
+    "AnalysisSnapshot",
     "ColumnResult",
     "analyze",
     "plan",
     "compare",
     "infer_contract",
     "validate",
+    "create_snapshot",
+    "load_snapshot",
+    "compare_snapshots",
     "__version__",
 ]

From 95e86954d67cc56b623974421d5c4a076c921373 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:11:25 +0530
Subject: [PATCH 025/439] feat: add AnalysisResult snapshot helper

---
 src/framevitals/result.py | 33 ++++++++++++---------------------
 1 file changed, 12 insertions(+), 21 deletions(-)

diff --git a/src/framevitals/result.py b/src/framevitals/result.py
index 99584bc..9926d69 100644
--- a/src/framevitals/result.py
+++ b/src/framevitals/result.py
@@ -16,11 +16,7 @@
 
 
 class ColumnResult(dict):
-    """Structured view of one analyzed column.
-
-    The object is also a normal dictionary, making it easy to serialize or pass
-    to existing code while exposing common fields through attribute access.
-    """
+    """Structured view of one analyzed column."""
 
     def __getattr__(self, name: str) -> Any:
         try:
@@ -30,15 +26,7 @@ def __getattr__(self, name: str) -> Any:
 
 
 class AnalysisResult(dict):
-    """Backward-compatible result returned by :func:`framevitals.analyze`.
-
-    Existing code can continue using dictionary access::
-
-        result["health"]["overall_score"]
-
-    while newer code can use helpers such as ``summary()``, ``column()``,
-    ``to_json()``, ``to_html()``, and ``summary_text()``.
-    """
+    """Backward-compatible result returned by :func:`framevitals.analyze`."""
 
     schema_version = "1"
 
@@ -122,13 +110,7 @@ def summary(self) -> dict[str, Any]:
         }
 
     def column(self, name: str) -> ColumnResult:
-        """Return the combined profile/role view for a single column.
-
-        Raises
-        ------
-        KeyError
-            If the requested column was not part of the analyzed dataset.
-        """
+        """Return the combined profile/role view for a single column."""
         profile = self.get("profile", {})
         roles = self.get("column_roles", {})
         if not isinstance(profile, dict):
@@ -206,6 +188,15 @@ def to_html(self, destination: str | Path | None = None) -> str | Path:
         path.write_text(rendered, encoding="utf-8")
         return path
 
+    def snapshot(self, destination: str | Path | None = None):
+        """Create a compact versioned monitoring snapshot from this result."""
+        from framevitals.snapshots import create_snapshot
+
+        snapshot = create_snapshot(self)
+        if destination is not None:
+            snapshot.to_json(destination)
+        return snapshot
+
     def _repr_html_(self) -> str:
         """Provide a compact rich representation in Jupyter-compatible clients."""
         from framevitals.reporting.html import render_notebook_summary

From 4936cdf9794b30a1f65079dcca460daeb92c12ac Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:11:40 +0530
Subject: [PATCH 026/439] test: cover snapshot creation loading and diffs

---
 tests/test_snapshots.py | 91 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 91 insertions(+)
 create mode 100644 tests/test_snapshots.py

diff --git a/tests/test_snapshots.py b/tests/test_snapshots.py
new file mode 100644
index 0000000..7a0c39b
--- /dev/null
+++ b/tests/test_snapshots.py
@@ -0,0 +1,91 @@
+import json
+
+from framevitals.result import AnalysisResult
+from framevitals.snapshots import compare_snapshots, load_snapshot
+
+
+def _result(*, columns=None, health=80.0, missing=None, findings=None):
+    columns = columns or {"age": "int64", "city": "object"}
+    missing = missing or {name: 0.0 for name in columns}
+    return AnalysisResult({
+        "dataset_id": "fv_runtime_id",
+        "filename": "customers.csv",
+        "analysis_mode": "quick",
+        "profile": {
+            "shape": {"rows": 10, "columns": len(columns)},
+            "columns": list(columns),
+            "dtypes": columns,
+            "missing_percent": missing,
+            "duplicate_percent": 0.0,
+            "memory_usage_mb": 0.1,
+        },
+        "health": {
+            "overall_score": health,
+            "label": "Good",
+            "components": {"completeness": health},
+        },
+        "ml_readiness": {
+            "score": 75.0,
+            "label": "Mostly Ready",
+            "issues": {"missing_percent": max(missing.values(), default=0.0)},
+        },
+        "signals": [],
+        "findings": findings or [],
+        "config": {
+            "mode": "quick",
+            "target": None,
+            "artifacts": False,
+            "workers": 2,
+        },
+    })
+
+
+def test_snapshot_is_compact_deterministic_state(tmp_path):
+    first = _result().snapshot()
+    second = _result().snapshot()
+
+    assert first["snapshot_schema_version"] == "1"
+    assert first["fingerprint"] == second["fingerprint"]
+    assert "profile" not in first
+    assert first["state"]["dataset"]["dtypes"]["age"] == "int64"
+
+    path = tmp_path / "baseline.json"
+    returned = _result().snapshot(path)
+    assert path.exists()
+    assert returned["fingerprint"] == load_snapshot(path)["fingerprint"]
+
+
+def test_snapshot_diff_reports_schema_health_missingness_and_findings():
+    reference = _result(
+        health=85.0,
+        findings=[{"code": "signal.duplicate_records"}],
+    ).snapshot()
+    current = _result(
+        columns={"age": "float64", "city": "object", "segment": "object"},
+        health=76.5,
+        missing={"age": 10.0, "city": 0.0, "segment": 5.0},
+        findings=[{"code": "signal.data_completeness"}],
+    ).snapshot()
+
+    diff = compare_snapshots(reference, current)
+
+    assert diff["changed"] is True
+    assert diff["schema"]["added_columns"] == ["segment"]
+    assert diff["schema"]["type_changes"]["age"] == {
+        "reference": "int64",
+        "current": "float64",
+    }
+    assert diff["missingness_changes"]["age"]["delta"] == 10.0
+    assert diff["health_delta"] == -8.5
+    assert diff["findings"]["new"] == ["signal.data_completeness"]
+    assert diff["findings"]["resolved"] == ["signal.duplicate_records"]
+
+
+def test_snapshot_json_roundtrip(tmp_path):
+    path = tmp_path / "snapshot.json"
+    snapshot = _result().snapshot(path)
+
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    assert payload["fingerprint"] == snapshot["fingerprint"]
+    loaded = load_snapshot(path)
+    assert loaded.diff(snapshot)["changed"] is False

From 305b499d9d68f9293ad1a30af92b732f60437db4 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:12:24 +0530
Subject: [PATCH 027/439] feat: add bounded semantic type inference

---
 src/framevitals/semantic_types.py | 131 ++++++++++++++++++++++++++++++
 1 file changed, 131 insertions(+)
 create mode 100644 src/framevitals/semantic_types.py

diff --git a/src/framevitals/semantic_types.py b/src/framevitals/semantic_types.py
new file mode 100644
index 0000000..c3582a5
--- /dev/null
+++ b/src/framevitals/semantic_types.py
@@ -0,0 +1,131 @@
+"""Deterministic semantic type inference for text-like columns.
+
+The detector intentionally uses bounded samples so semantic typing remains
+cheap on large datasets. It augments, rather than replaces, dtype and name-based
+column role inference.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import json
+import re
+import uuid
+from typing import Callable
+
+import pandas as pd
+
+
+_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
+_URL_RE = re.compile(r"^(?:https?://|www\.)[^\s]+$", re.IGNORECASE)
+_PHONE_RE = re.compile(r"^\+?[0-9][0-9\s().-]{6,}$")
+_PERCENT_RE = re.compile(r"^[+-]?(?:\d+(?:\.\d+)?|\.\d+)\s*%$")
+_CURRENCY_RE = re.compile(
+    r"^(?:[$€£₹¥]\s*[+-]?[\d,.]+|(?:USD|EUR|GBP|INR|JPY)\s+[+-]?[\d,.]+)$",
+    re.IGNORECASE,
+)
+_BOOL_TOKEN_RE = re.compile(
+    r"^(?:true|false|yes|no|y|n|on|off)$",
+    re.IGNORECASE,
+)
+
+
+def _matches_uuid(value: str) -> bool:
+    try:
+        uuid.UUID(value)
+        return True
+    except (ValueError, AttributeError, TypeError):
+        return False
+
+
+def _matches_ip(value: str) -> bool:
+    try:
+        ipaddress.ip_address(value)
+        return True
+    except ValueError:
+        return False
+
+
+def _matches_phone(value: str) -> bool:
+    if not _PHONE_RE.match(value):
+        return False
+    digit_count = sum(character.isdigit() for character in value)
+    return 7 <= digit_count <= 15
+
+
+def _matches_json(value: str) -> bool:
+    stripped = value.strip()
+    if not stripped.startswith(("{", "[")):
+        return False
+    try:
+        parsed = json.loads(stripped)
+    except (json.JSONDecodeError, TypeError):
+        return False
+    return isinstance(parsed, (dict, list))
+
+
+def _ratio(sample: list[str], predicate: Callable[[str], bool]) -> float:
+    if not sample:
+        return 0.0
+    return sum(1 for value in sample if predicate(value)) / len(sample)
+
+
+def infer_semantic_types(
+    series: pd.Series,
+    *,
+    max_samples: int = 100,
+    threshold: float = 0.70,
+) -> dict:
+    """Infer semantic value types from a bounded non-null sample.
+
+    Returns a dictionary with ``primary``, ranked ``candidates``, and
+    ``sample_size``. Empty/unsupported columns return no candidates.
+    """
+    if max_samples < 1:
+        raise ValueError("max_samples must be at least 1.")
+    if not 0 < threshold <= 1:
+        raise ValueError("threshold must be in the interval (0, 1].")
+
+    is_text = (
+        pd.api.types.is_object_dtype(series)
+        or pd.api.types.is_string_dtype(series.dtype)
+        or isinstance(series.dtype, pd.CategoricalDtype)
+    )
+    if not is_text:
+        return {"primary": None, "candidates": [], "sample_size": 0}
+
+    sample = [
+        value.strip()
+        for value in series.dropna().astype(str).head(max_samples).tolist()
+        if value.strip()
+    ]
+    if not sample:
+        return {"primary": None, "candidates": [], "sample_size": 0}
+
+    checks: list[tuple[str, Callable[[str], bool]]] = [
+        ("email", lambda value: bool(_EMAIL_RE.match(value))),
+        ("url", lambda value: bool(_URL_RE.match(value))),
+        ("uuid", _matches_uuid),
+        ("ip_address", _matches_ip),
+        ("phone", _matches_phone),
+        ("percentage", lambda value: bool(_PERCENT_RE.match(value))),
+        ("currency", lambda value: bool(_CURRENCY_RE.match(value))),
+        ("json", _matches_json),
+        ("boolean_token", lambda value: bool(_BOOL_TOKEN_RE.match(value))),
+    ]
+
+    candidates = []
+    for semantic_type, predicate in checks:
+        confidence = _ratio(sample, predicate)
+        if confidence >= threshold:
+            candidates.append({
+                "type": semantic_type,
+                "confidence": round(confidence, 4),
+            })
+
+    candidates.sort(key=lambda item: (-item["confidence"], item["type"]))
+    return {
+        "primary": candidates[0]["type"] if candidates else None,
+        "candidates": candidates,
+        "sample_size": len(sample),
+    }

From 0f722c63564d70131fff53b868393f593452af4a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:12:50 +0530
Subject: [PATCH 028/439] feat: integrate semantic value types into column
 roles

---
 src/framevitals/column_roles.py | 55 ++++++++++++++++++++++++++-------
 1 file changed, 44 insertions(+), 11 deletions(-)

diff --git a/src/framevitals/column_roles.py b/src/framevitals/column_roles.py
index 21801b5..c8d1cd6 100644
--- a/src/framevitals/column_roles.py
+++ b/src/framevitals/column_roles.py
@@ -1,17 +1,14 @@
 """
 Column Role Inference Engine
 ============================
-Assigns semantic roles to each column based on name keywords,
-dtype, unique ratio, missingness, and statistical properties.
-
-Each column receives a SET of roles (not a single label),
-enabling downstream modules to make informed decisions.
+Assigns semantic roles to each column based on name keywords, dtype, unique
+ratio, missingness, statistical properties, and bounded value-pattern samples.
 """
 
-import re
-
 import pandas as pd
 
+from framevitals.semantic_types import infer_semantic_types
+
 ID_KEYWORDS = [
     "id", "uuid", "hash", "key", "identifier", "index",
     "roll", "roll_number", "rollno", "roll number",
@@ -56,16 +53,24 @@
     "pass", "fail", "approved", "rejected",
 ]
 
+SEMANTIC_ROLE_MAP = {
+    "email": "email_like",
+    "url": "url_like",
+    "uuid": "uuid_like",
+    "ip_address": "ip_address_like",
+    "phone": "phone_like",
+    "percentage": "percentage_like",
+    "currency": "currency_like",
+    "json": "json_like",
+    "boolean_token": "boolean_token_like",
+}
+
 
 def _name_matches(column_name: str, keywords: list) -> bool:
     lower = column_name.lower().replace("-", "_")
     return any(kw in lower for kw in keywords)
 
 
-def _safe_unique_ratio(series: pd.Series) -> float:
-    return series.nunique(dropna=True) / max(len(series), 1)
-
-
 def _classify_missingness(missing_percent: float) -> str:
     if missing_percent >= 90:
         return "severe_missing"
@@ -96,6 +101,13 @@ def _infer_single_column_roles(column: str, series: pd.Series, rows: int) -> dic
         or isinstance(series.dtype, pd.CategoricalDtype)
     )
 
+    semantic = (
+        infer_semantic_types(series)
+        if is_text
+        else {"primary": None, "candidates": [], "sample_size": 0}
+    )
+    semantic_primary = semantic.get("primary")
+
     if is_numeric:
         roles.add("numeric")
     if is_bool:
@@ -131,6 +143,15 @@ def _infer_single_column_roles(column: str, series: pd.Series, rows: int) -> dic
     if _name_matches(column, TARGET_HINT_KEYWORDS):
         roles.add("target_hint")
 
+    if semantic_primary in SEMANTIC_ROLE_MAP:
+        roles.add(SEMANTIC_ROLE_MAP[semantic_primary])
+    if semantic_primary in {"email", "phone", "ip_address"}:
+        roles.add("sensitive")
+    if semantic_primary == "uuid":
+        roles.add("id_like")
+    if semantic_primary == "currency":
+        roles.add("price_like")
+
     if not is_numeric and not is_bool:
         sample = series.dropna().astype(str).head(30)
         if len(sample) > 0:
@@ -169,6 +190,9 @@ def _infer_single_column_roles(column: str, series: pd.Series, rows: int) -> dic
         "non_missing_count": non_missing,
         "is_numeric": bool(is_numeric),
         "is_categorical": bool(is_text or is_bool),
+        "semantic_type": semantic_primary,
+        "semantic_candidates": semantic.get("candidates", []),
+        "semantic_sample_size": int(semantic.get("sample_size", 0)),
     }
 
 
@@ -234,6 +258,15 @@ def summarize_roles(column_roles: dict) -> dict:
             column_roles, "regression_target_candidate"
         ),
         "sensitive": get_columns_with_role(column_roles, "sensitive"),
+        "email_like": get_columns_with_role(column_roles, "email_like"),
+        "url_like": get_columns_with_role(column_roles, "url_like"),
+        "uuid_like": get_columns_with_role(column_roles, "uuid_like"),
+        "ip_address_like": get_columns_with_role(column_roles, "ip_address_like"),
+        "phone_like": get_columns_with_role(column_roles, "phone_like"),
+        "percentage_like": get_columns_with_role(column_roles, "percentage_like"),
+        "currency_like": get_columns_with_role(column_roles, "currency_like"),
+        "json_like": get_columns_with_role(column_roles, "json_like"),
+        "boolean_token_like": get_columns_with_role(column_roles, "boolean_token_like"),
         "high_missing": [
             col for col, info in column_roles.items()
             if info["missing_percent"] >= 20

From 995374a2269a4d12d64905edaa916a3c4422f564 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:13:54 +0530
Subject: [PATCH 029/439] feat: expose semantic dataset signals without
 rescanning text

---
 src/framevitals/dataset_signals.py | 74 ++++++++++++++----------------
 1 file changed, 34 insertions(+), 40 deletions(-)

diff --git a/src/framevitals/dataset_signals.py b/src/framevitals/dataset_signals.py
index 6782e20..950e971 100644
--- a/src/framevitals/dataset_signals.py
+++ b/src/framevitals/dataset_signals.py
@@ -1,16 +1,13 @@
 """
 Dataset Signal Detector
 ========================
-Produces a flat dictionary of boolean + numeric signals describing
-the structural characteristics of the dataset.
+Produces a flat dictionary of boolean + numeric signals describing the
+structural characteristics of the dataset.
 
-These signals drive the analysis selector engine — they answer
-questions like "does this dataset have missing values?" or
-"are there ID-like columns?" without hardcoding domain logic.
+Signals reuse the role map when available so semantic/type scans are performed
+once per pipeline run rather than independently by downstream modules.
 """
 
-import re
-
 import pandas as pd
 
 from framevitals.column_roles import (
@@ -20,33 +17,6 @@
 )
 
 
-_EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
-_TEXT_DTYPES = ["object", "string", "category"]
-
-
-def _detect_long_text_columns(df: pd.DataFrame) -> list:
-    result = []
-    for col in df.select_dtypes(include=_TEXT_DTYPES).columns:
-        lengths = df[col].dropna().astype(str).str.len()
-        if len(lengths) == 0:
-            continue
-        if float(lengths.mean()) > 50 or int(lengths.max()) > 200:
-            result.append(col)
-    return result
-
-
-def _detect_email_columns(df: pd.DataFrame) -> list:
-    result = []
-    for col in df.select_dtypes(include=_TEXT_DTYPES).columns:
-        sample = df[col].dropna().astype(str).head(100)
-        if sample.empty:
-            continue
-        match_ratio = sample.apply(lambda v: bool(_EMAIL_PATTERN.match(v))).mean()
-        if match_ratio >= 0.5:
-            result.append(col)
-    return result
-
-
 def _detect_potential_leakage(df: pd.DataFrame, column_roles: dict) -> bool:
     """Return whether a very high non-ID numeric correlation suggests leakage."""
     meaningful = get_meaningful_numeric_columns(df, column_roles)
@@ -73,7 +43,7 @@ def detect_dataset_signals(
 
     ``column_roles`` is optional for backward compatibility. The main analysis
     pipeline passes its already-computed role map so this stage does not repeat
-    the most expensive per-column semantic scan.
+    per-column semantic scans.
     """
     rows, cols = df.shape
 
@@ -108,16 +78,24 @@ def detect_dataset_signals(
     high_card = get_columns_with_role(column_roles, "high_cardinality")
     unique_like = get_columns_with_role(column_roles, "unique_like")
     target_candidates = get_columns_with_role(column_roles, "target_candidate")
-
-    # Column-role inference already computes the same full-column text-length
-    # rule, so reuse it instead of scanning text columns a second time.
     long_text = get_columns_with_role(column_roles, "long_text")
-    email_like = _detect_email_columns(df)
+
+    # Semantic/value-pattern roles are already computed during role inference.
+    # Reusing them here avoids additional text-column samples/scans.
+    email_like = get_columns_with_role(column_roles, "email_like")
+    url_like = get_columns_with_role(column_roles, "url_like")
+    uuid_like = get_columns_with_role(column_roles, "uuid_like")
+    ip_address_like = get_columns_with_role(column_roles, "ip_address_like")
+    phone_like = get_columns_with_role(column_roles, "phone_like")
+    percentage_like = get_columns_with_role(column_roles, "percentage_like")
+    currency_like = get_columns_with_role(column_roles, "currency_like")
+    json_like = get_columns_with_role(column_roles, "json_like")
+    boolean_token_like = get_columns_with_role(column_roles, "boolean_token_like")
+
     has_leakage_risk = _detect_potential_leakage(df, column_roles)
 
     lower_map = {c.lower(): c for c in df.columns}
     has_bid_ask = "bid" in lower_map and "ask" in lower_map
-
     has_time_series = len(time_like) > 0 and len(numeric_cols) >= 1 and rows >= 20
 
     return {
@@ -149,6 +127,14 @@ def detect_dataset_signals(
         "has_time_series_structure": has_time_series,
         "has_sensitive_column_candidates": len(sensitive) > 0,
         "has_email_like_columns": len(email_like) > 0,
+        "has_url_like_columns": len(url_like) > 0,
+        "has_uuid_like_columns": len(uuid_like) > 0,
+        "has_ip_address_like_columns": len(ip_address_like) > 0,
+        "has_phone_like_columns": len(phone_like) > 0,
+        "has_percentage_like_columns": len(percentage_like) > 0,
+        "has_currency_like_columns": len(currency_like) > 0,
+        "has_json_like_columns": len(json_like) > 0,
+        "has_boolean_token_like_columns": len(boolean_token_like) > 0,
         "has_potential_leakage": has_leakage_risk,
         "has_target_candidates": len(target_candidates) > 0,
         "is_small_dataset": rows < 200,
@@ -163,6 +149,14 @@ def detect_dataset_signals(
         "sensitive_columns": sensitive,
         "constant_columns": constant,
         "email_like_columns": email_like,
+        "url_like_columns": url_like,
+        "uuid_like_columns": uuid_like,
+        "ip_address_like_columns": ip_address_like,
+        "phone_like_columns": phone_like,
+        "percentage_like_columns": percentage_like,
+        "currency_like_columns": currency_like,
+        "json_like_columns": json_like,
+        "boolean_token_like_columns": boolean_token_like,
         "long_text_columns": long_text,
         "binary_columns": binary,
         "low_cardinality_columns": low_card,

From a38d9247cb91b7e92878993edbcc0eb21e6cbe7c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:14:15 +0530
Subject: [PATCH 030/439] feat: surface semantic types in column results

---
 src/framevitals/result.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/result.py b/src/framevitals/result.py
index 9926d69..7902f63 100644
--- a/src/framevitals/result.py
+++ b/src/framevitals/result.py
@@ -110,7 +110,7 @@ def summary(self) -> dict[str, Any]:
         }
 
     def column(self, name: str) -> ColumnResult:
-        """Return the combined profile/role view for a single column."""
+        """Return the combined profile, role, and semantic view for one column."""
         profile = self.get("profile", {})
         roles = self.get("column_roles", {})
         if not isinstance(profile, dict):
@@ -140,6 +140,9 @@ def column(self, name: str) -> ColumnResult:
             "name": name,
             "dtype": profile.get("dtypes", {}).get(name),
             "roles": list(role_info.get("roles", [])),
+            "semantic_type": role_info.get("semantic_type"),
+            "semantic_candidates": list(role_info.get("semantic_candidates", [])),
+            "semantic_sample_size": role_info.get("semantic_sample_size", 0),
             "missing_count": profile.get("missing_counts", {}).get(name),
             "missing_percent": profile.get("missing_percent", {}).get(name),
             "unique_count": role_info.get("unique_count"),

From 5ff2eb17ca8793e98e6cbff12d1b97cc1466744f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:14:36 +0530
Subject: [PATCH 031/439] test: cover semantic type inference and dataset
 signals

---
 tests/test_semantic_types.py | 128 +++++++++++++++++++++++++++++++++++
 1 file changed, 128 insertions(+)
 create mode 100644 tests/test_semantic_types.py

diff --git a/tests/test_semantic_types.py b/tests/test_semantic_types.py
new file mode 100644
index 0000000..fafabcc
--- /dev/null
+++ b/tests/test_semantic_types.py
@@ -0,0 +1,128 @@
+import pandas as pd
+
+import framevitals
+from framevitals.column_roles import infer_column_roles, summarize_roles
+from framevitals.dataset_signals import detect_dataset_signals
+from framevitals.profiler import build_profile
+from framevitals.semantic_types import infer_semantic_types
+
+
+def test_semantic_type_detector_recognizes_common_value_patterns():
+    cases = {
+        "email": pd.Series(["a@example.com", "b@example.com", "c@example.org"]),
+        "url": pd.Series(["https://example.com/a", "http://openai.com", "www.python.org"]),
+        "uuid": pd.Series([
+            "550e8400-e29b-41d4-a716-446655440000",
+            "6ba7b810-9dad-11d1-80b4-00c04fd430c8",
+            "6ba7b811-9dad-11d1-80b4-00c04fd430c8",
+        ]),
+        "ip_address": pd.Series(["192.168.1.1", "10.0.0.8", "2001:db8::1"]),
+        "phone": pd.Series(["+91 98765 43210", "020-1234-5678", "+1 (415) 555-0100"]),
+        "percentage": pd.Series(["12%", "3.5%", "-0.5%"]),
+        "currency": pd.Series(["₹1,200", "$19.99", "INR 5000"]),
+        "json": pd.Series(['{"a": 1}', '[1, 2]', '{"ok": true}']),
+        "boolean_token": pd.Series(["yes", "NO", "true", "off"]),
+    }
+
+    for expected, series in cases.items():
+        result = infer_semantic_types(series)
+        assert result["primary"] == expected
+        assert result["candidates"][0]["confidence"] >= 0.7
+
+
+def test_semantic_type_detector_is_bounded_and_ignores_numeric_columns():
+    values = [f"user{index}@example.com" for index in range(250)]
+    text_result = infer_semantic_types(pd.Series(values), max_samples=40)
+    numeric_result = infer_semantic_types(pd.Series([1, 2, 3, 4]))
+
+    assert text_result["primary"] == "email"
+    assert text_result["sample_size"] == 40
+    assert numeric_result == {
+        "primary": None,
+        "candidates": [],
+        "sample_size": 0,
+    }
+
+
+def test_value_semantics_augment_column_roles_without_name_hints():
+    df = pd.DataFrame({
+        "contact_value": [
+            "a@example.com",
+            "b@example.com",
+            "c@example.com",
+            "d@example.com",
+        ],
+        "record_key": [
+            "550e8400-e29b-41d4-a716-446655440000",
+            "6ba7b810-9dad-11d1-80b4-00c04fd430c8",
+            "6ba7b811-9dad-11d1-80b4-00c04fd430c8",
+            "6ba7b812-9dad-11d1-80b4-00c04fd430c8",
+        ],
+        "web": [
+            "https://example.com/a",
+            "https://example.com/b",
+            "https://example.com/c",
+            "https://example.com/d",
+        ],
+    })
+
+    roles = infer_column_roles(df)
+
+    assert roles["contact_value"]["semantic_type"] == "email"
+    assert "email_like" in roles["contact_value"]["roles"]
+    assert "sensitive" in roles["contact_value"]["roles"]
+
+    assert roles["record_key"]["semantic_type"] == "uuid"
+    assert "uuid_like" in roles["record_key"]["roles"]
+    assert "id_like" in roles["record_key"]["roles"]
+
+    assert roles["web"]["semantic_type"] == "url"
+    assert "url_like" in roles["web"]["roles"]
+
+    summary = summarize_roles(roles)
+    assert summary["email_like"] == ["contact_value"]
+    assert summary["uuid_like"] == ["record_key"]
+    assert summary["url_like"] == ["web"]
+
+
+def test_dataset_signals_reuse_semantic_roles():
+    df = pd.DataFrame({
+        "contact": ["a@example.com", "b@example.com", "c@example.com"],
+        "homepage": ["https://a.example", "https://b.example", "https://c.example"],
+        "host": ["10.0.0.1", "10.0.0.2", "10.0.0.3"],
+        "price_text": ["₹100", "₹200", "₹300"],
+    })
+    profile = build_profile(df)
+    roles = infer_column_roles(df)
+
+    signals = detect_dataset_signals(df, profile, column_roles=roles)
+
+    assert signals["has_email_like_columns"] is True
+    assert signals["email_like_columns"] == ["contact"]
+    assert signals["has_url_like_columns"] is True
+    assert signals["url_like_columns"] == ["homepage"]
+    assert signals["has_ip_address_like_columns"] is True
+    assert signals["ip_address_like_columns"] == ["host"]
+    assert signals["has_currency_like_columns"] is True
+    assert signals["currency_like_columns"] == ["price_text"]
+    assert set(signals["sensitive_columns"]) >= {"contact", "host"}
+
+
+def test_column_result_surfaces_semantic_information():
+    df = pd.DataFrame({
+        "contact": [
+            "a@example.com",
+            "b@example.com",
+            "c@example.com",
+            "d@example.com",
+        ],
+        "value": [1, 2, 3, 4],
+    })
+
+    result = framevitals.analyze(df, mode="quick")
+    column = result.column("contact")
+
+    assert column.semantic_type == "email"
+    assert column.semantic_sample_size == 4
+    assert column.semantic_candidates[0]["type"] == "email"
+    assert "email_like" in column.roles

From 6fd2bd7ff200d55d7a6aaf184dd20abcb0ae9c0b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:16:23 +0530
Subject: [PATCH 032/439] feat: add unified target intelligence analysis

---
 src/framevitals/target_intelligence.py | 335 +++++++++++++++++++++++++
 1 file changed, 335 insertions(+)
 create mode 100644 src/framevitals/target_intelligence.py

diff --git a/src/framevitals/target_intelligence.py b/src/framevitals/target_intelligence.py
new file mode 100644
index 0000000..5f510de
--- /dev/null
+++ b/src/framevitals/target_intelligence.py
@@ -0,0 +1,335 @@
+"""Unified target-aware diagnostics for supervised-learning datasets.
+
+This layer connects FrameVitals' existing target and leakage diagnostics and
+adds lightweight, interpretable feature-to-target association ranking. It is
+not an AutoML trainer and does not fit production models.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+from framevitals.column_roles import infer_column_roles
+from framevitals.target_analyzer import analyze_target
+from framevitals.target_leakage import run_target_leakage_analysis
+
+
+_EXCLUDED_ROLES = {"id_like", "constant"}
+
+
+def _safe_float(value: Any) -> float | None:
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        return None
+    if not math.isfinite(number):
+        return None
+    return round(number, 6)
+
+
+def _cramers_v(feature: pd.Series, target: pd.Series) -> tuple[float | None, int]:
+    mask = feature.notna() & target.notna()
+    n = int(mask.sum())
+    if n < 10:
+        return None, n
+
+    table = pd.crosstab(feature[mask], target[mask])
+    if table.shape[0] < 2 or table.shape[1] < 2:
+        return None, n
+    if table.shape[0] > 50 or table.shape[1] > 50:
+        return None, n
+
+    try:
+        chi2, _, _, _ = stats.chi2_contingency(table, correction=False)
+    except ValueError:
+        return None, n
+
+    phi2 = chi2 / n
+    rows, cols = table.shape
+    denominator = min(rows - 1, cols - 1)
+    if denominator <= 0:
+        return None, n
+    return _safe_float(math.sqrt(max(phi2, 0.0) / denominator)), n
+
+
+def _correlation_ratio(categories: pd.Series, values: pd.Series) -> tuple[float | None, int]:
+    mask = categories.notna() & values.notna()
+    n = int(mask.sum())
+    if n < 10:
+        return None, n
+
+    groups = categories[mask]
+    numeric = pd.to_numeric(values[mask], errors="coerce")
+    valid = numeric.notna()
+    groups = groups[valid]
+    numeric = numeric[valid]
+    n = len(numeric)
+    if n < 10 or groups.nunique(dropna=True) < 2 or groups.nunique(dropna=True) > 50:
+        return None, n
+
+    grand_mean = float(numeric.mean())
+    total = float(((numeric - grand_mean) ** 2).sum())
+    if total <= 0:
+        return None, n
+
+    between = 0.0
+    for _, group_values in numeric.groupby(groups):
+        if group_values.empty:
+            continue
+        between += len(group_values) * (float(group_values.mean()) - grand_mean) ** 2
+
+    eta_squared = max(0.0, min(1.0, between / total))
+    return _safe_float(math.sqrt(eta_squared)), n
+
+
+def _numeric_association(feature: pd.Series, target: pd.Series) -> tuple[float | None, int, str]:
+    mask = feature.notna() & target.notna()
+    n = int(mask.sum())
+    if n < 10:
+        return None, n, "spearman"
+
+    x = pd.to_numeric(feature[mask], errors="coerce")
+    y = pd.to_numeric(target[mask], errors="coerce")
+    valid = x.notna() & y.notna()
+    x = x[valid]
+    y = y[valid]
+    n = len(x)
+    if n < 10 or x.nunique() < 2 or y.nunique() < 2:
+        return None, n, "spearman"
+
+    try:
+        coefficient, _ = stats.spearmanr(x, y)
+    except Exception:
+        return None, n, "spearman"
+    return _safe_float(abs(coefficient)), n, "spearman"
+
+
+def _binary_numeric_association(
+    feature: pd.Series,
+    target: pd.Series,
+) -> tuple[float | None, int, str]:
+    mask = feature.notna() & target.notna()
+    n = int(mask.sum())
+    if n < 10:
+        return None, n, "point_biserial"
+
+    classes = list(pd.unique(target[mask]))
+    if len(classes) != 2:
+        score, n_eta = _correlation_ratio(target, feature)
+        return score, n_eta, "correlation_ratio"
+
+    encoded = target[mask].map({classes[0]: 0, classes[1]: 1})
+    numeric = pd.to_numeric(feature[mask], errors="coerce")
+    valid = numeric.notna() & encoded.notna()
+    numeric = numeric[valid]
+    encoded = encoded[valid]
+    n = len(numeric)
+    if n < 10 or numeric.nunique() < 2:
+        return None, n, "point_biserial"
+
+    try:
+        coefficient, _ = stats.pointbiserialr(encoded.astype(float), numeric.astype(float))
+    except Exception:
+        return None, n, "point_biserial"
+    return _safe_float(abs(coefficient)), n, "point_biserial"
+
+
+def _association_strength(score: float | None) -> str:
+    if score is None:
+        return "unknown"
+    if score >= 0.8:
+        return "very_strong"
+    if score >= 0.6:
+        return "strong"
+    if score >= 0.4:
+        return "moderate"
+    if score >= 0.2:
+        return "weak"
+    return "very_weak"
+
+
+def _rank_target_associations(
+    df: pd.DataFrame,
+    *,
+    target_column: str,
+    task_type: str,
+    column_roles: dict,
+    max_features: int,
+) -> list[dict[str, Any]]:
+    target = df[target_column]
+    target_is_numeric = pd.api.types.is_numeric_dtype(target)
+    associations: list[dict[str, Any]] = []
+
+    for column in df.columns:
+        if column == target_column:
+            continue
+
+        role_info = column_roles.get(column, {})
+        role_set = set(role_info.get("roles", []))
+        if role_set.intersection(_EXCLUDED_ROLES):
+            continue
+
+        feature = df[column]
+        feature_is_numeric = pd.api.types.is_numeric_dtype(feature)
+        score: float | None = None
+        overlap = 0
+        method = "unsupported"
+
+        if task_type == "regression" and target_is_numeric:
+            if feature_is_numeric:
+                score, overlap, method = _numeric_association(feature, target)
+            else:
+                score, overlap = _correlation_ratio(feature, target)
+                method = "correlation_ratio"
+        elif task_type == "classification":
+            if feature_is_numeric:
+                score, overlap, method = _binary_numeric_association(feature, target)
+            else:
+                score, overlap = _cramers_v(feature, target)
+                method = "cramers_v"
+
+        if score is None:
+            continue
+
+        associations.append({
+            "feature": column,
+            "score": score,
+            "strength": _association_strength(score),
+            "method": method,
+            "overlap": int(overlap),
+        })
+
+    associations.sort(key=lambda item: (-item["score"], item["feature"]))
+    return associations[:max_features]
+
+
+def _split_guidance(
+    target_profile: dict[str, Any],
+    column_roles: dict,
+) -> dict[str, Any]:
+    time_like = [
+        column
+        for column, info in column_roles.items()
+        if "time_like" in info.get("roles", [])
+    ]
+    task_type = target_profile.get("task_type")
+    details = target_profile.get("details", {}) or {}
+
+    if time_like:
+        return {
+            "strategy": "review_time_aware_split",
+            "reason": (
+                "Time-like columns are present. If rows represent chronological "
+                "events, prefer an ordered/time-based split to avoid future-to-past leakage."
+            ),
+            "time_candidates": time_like[:10],
+        }
+
+    if task_type == "classification" and details.get("class_count", 0) >= 2:
+        return {
+            "strategy": "stratified_random_split",
+            "reason": "Preserve target-class proportions across train/validation/test splits.",
+            "time_candidates": [],
+        }
+
+    if task_type == "regression":
+        return {
+            "strategy": "random_split",
+            "reason": (
+                "No time-like structure was detected; a reproducible random split is a "
+                "reasonable baseline for regression."
+            ),
+            "time_candidates": [],
+        }
+
+    return {
+        "strategy": "review_manually",
+        "reason": "FrameVitals could not infer a safe split strategy from the target metadata.",
+        "time_candidates": [],
+    }
+
+
+def run_target_intelligence(
+    df: pd.DataFrame,
+    *,
+    target_column: str | None,
+    column_roles: dict | None = None,
+    max_features: int = 25,
+) -> dict[str, Any]:
+    """Run explainable target-quality, leakage, association, and split checks."""
+    if not target_column or target_column not in df.columns:
+        return {
+            "available": False,
+            "message": "No valid target column selected.",
+        }
+    if max_features < 1:
+        raise ValueError("max_features must be at least 1.")
+
+    if column_roles is None:
+        column_roles = infer_column_roles(df)
+
+    target_profile = analyze_target(df, target_column)
+    leakage = run_target_leakage_analysis(df, target_column)
+    task_type = target_profile.get("task_type", "unknown")
+    associations = _rank_target_associations(
+        df,
+        target_column=target_column,
+        task_type=task_type,
+        column_roles=column_roles,
+        max_features=max_features,
+    )
+
+    target_roles = column_roles.get(target_column, {}).get("roles", [])
+    warnings: list[dict[str, str]] = []
+
+    if "id_like" in target_roles:
+        warnings.append({
+            "code": "target.id_like",
+            "severity": "high",
+            "message": "Selected target looks identifier-like and may not be a meaningful prediction target.",
+        })
+    if target_profile.get("missing_percent", 0) >= 20:
+        warnings.append({
+            "code": "target.high_missingness",
+            "severity": "high",
+            "message": f"Target has {target_profile['missing_percent']}% missing values.",
+        })
+
+    details = target_profile.get("details", {}) or {}
+    if task_type == "classification" and details.get("class_count", 0) > 50:
+        warnings.append({
+            "code": "target.high_cardinality_classification",
+            "severity": "medium",
+            "message": (
+                f"Target has {details['class_count']} classes; confirm this is intentional "
+                "before treating the task as classification."
+            ),
+        })
+
+    for item in leakage.get("warnings", []):
+        warnings.append({
+            "code": f"target.leakage.{item.get('feature', 'feature')}",
+            "severity": str(item.get("risk", "medium")).lower(),
+            "message": str(item.get("reason", "Potential target leakage detected.")),
+        })
+
+    severity_order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
+    warnings.sort(key=lambda item: (severity_order.get(item["severity"], 9), item["code"]))
+
+    return {
+        "available": True,
+        "target_column": target_column,
+        "task_type": task_type,
+        "target_profile": target_profile,
+        "target_roles": list(target_roles),
+        "split_guidance": _split_guidance(target_profile, column_roles),
+        "leakage": leakage,
+        "top_associations": associations,
+        "warning_count": len(warnings),
+        "warnings": warnings,
+    }

From 833bdb1d1b674d2c4a96e626feddee8686ef8b5e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:17:01 +0530
Subject: [PATCH 033/439] feat: integrate target intelligence into analysis
 pipeline

---
 src/framevitals/pipeline.py | 44 +++++++++++++++----------------------
 1 file changed, 18 insertions(+), 26 deletions(-)

diff --git a/src/framevitals/pipeline.py b/src/framevitals/pipeline.py
index 85f8b35..0546122 100644
--- a/src/framevitals/pipeline.py
+++ b/src/framevitals/pipeline.py
@@ -30,6 +30,7 @@
 from framevitals.model_leaderboard import run_model_leaderboard
 from framevitals.profiler import build_profile
 from framevitals.signal_engine import build_signals
+from framevitals.target_intelligence import run_target_intelligence
 from framevitals.text_profile import profile_text_columns
 from framevitals.time_series import detect_and_analyze_time_series
 from framevitals.visualizer import generate_charts
@@ -63,31 +64,7 @@ def run_full_analysis(
     dataframe: pd.DataFrame | None = None,
     write_artifacts: bool = True,
 ) -> dict:
-    """Run the complete FrameVitals analysis pipeline.
-
-    Parameters
-    ----------
-    dataset_id:
-        Unique identifier used for optional artifact filenames.
-    file_path:
-        Path-like dataset source. Omit when ``dataframe`` is supplied.
-    original_filename:
-        Source label included in the result.
-    analysis_mode:
-        One of ``quick``, ``standard``, ``deep``, or ``research``.
-    skip_ai:
-        Skip the optional LLM report.
-    target_column:
-        Optional supervised-learning target column.
-    parallel_workers:
-        Worker count for independent diagnostic tasks.
-    dataframe:
-        Optional in-memory DataFrame. When supplied it takes precedence over
-        ``file_path`` and is copied before analysis.
-    write_artifacts:
-        Persist cleaned CSV/chart artifacts. Web application callers keep this
-        enabled; reusable library calls can disable filesystem side effects.
-    """
+    """Run the complete FrameVitals analysis pipeline."""
     overall_start = time.perf_counter()
     timings_ms: dict[str, Any] = {}
 
@@ -190,9 +167,23 @@ def run_full_analysis(
         ) * 1000
         timings_ms["phase3_tasks"] = per_task_ms
 
-    # Phase 4: target-aware ML chain.
+    # Phase 4: target-aware diagnostics and ML chain.
+    target_intelligence = None
     model_leaderboard = None
     explainability = None
+
+    if target_column:
+        t0 = time.perf_counter()
+        _, target_intelligence, _ = _safe_call(
+            "target_intelligence",
+            lambda: run_target_intelligence(
+                df,
+                target_column=target_column,
+                column_roles=column_roles,
+            ),
+        )
+        timings_ms["target_intelligence"] = (time.perf_counter() - t0) * 1000
+
     if target_column and analysis_mode in {"standard", "deep", "research"}:
         t0 = time.perf_counter()
         _, model_leaderboard, _ = _safe_call(
@@ -307,6 +298,7 @@ def run_full_analysis(
         "advanced": advanced,
         "deep_statistics_v2": deep_statistics_v2,
         "anomalies_v2": anomalies_v2,
+        "target_intelligence": target_intelligence,
         "model_leaderboard": model_leaderboard,
         "explainability": explainability,
         "time_series": time_series_analysis,

From b3a5d1735ff407dc18510cfee275cf3cff1c7680 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:17:20 +0530
Subject: [PATCH 034/439] test: cover unified target intelligence

---
 tests/test_target_intelligence_v2.py | 94 ++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)
 create mode 100644 tests/test_target_intelligence_v2.py

diff --git a/tests/test_target_intelligence_v2.py b/tests/test_target_intelligence_v2.py
new file mode 100644
index 0000000..806324e
--- /dev/null
+++ b/tests/test_target_intelligence_v2.py
@@ -0,0 +1,94 @@
+import pandas as pd
+
+import framevitals
+from framevitals.column_roles import infer_column_roles
+from framevitals.target_intelligence import run_target_intelligence
+
+
+def test_classification_target_intelligence_ranks_mixed_feature_types():
+    target = [0, 1] * 20
+    df = pd.DataFrame({
+        "numeric_signal": [value * 10 + (index % 3) for index, value in enumerate(target)],
+        "segment": ["stay" if value == 0 else "leave" for value in target],
+        "noise": list(range(40)),
+        "churn": target,
+    })
+
+    result = run_target_intelligence(
+        df,
+        target_column="churn",
+        column_roles=infer_column_roles(df),
+    )
+
+    assert result["available"] is True
+    assert result["task_type"] == "classification"
+    assert result["split_guidance"]["strategy"] == "stratified_random_split"
+
+    associations = {item["feature"]: item for item in result["top_associations"]}
+    assert associations["numeric_signal"]["method"] == "point_biserial"
+    assert associations["numeric_signal"]["score"] > 0.9
+    assert associations["segment"]["method"] == "cramers_v"
+    assert associations["segment"]["score"] > 0.9
+
+    leakage_features = {item["feature"] for item in result["leakage"]["warnings"]}
+    assert "segment" in leakage_features
+
+
+def test_regression_target_intelligence_uses_spearman_and_correlation_ratio():
+    target = list(range(1, 41))
+    df = pd.DataFrame({
+        "linear": [value * 3 for value in target],
+        "bucket": ["low"] * 20 + ["high"] * 20,
+        "target": target,
+    })
+
+    result = run_target_intelligence(df, target_column="target")
+    associations = {item["feature"]: item for item in result["top_associations"]}
+
+    assert result["task_type"] == "regression"
+    assert associations["linear"]["method"] == "spearman"
+    assert associations["linear"]["score"] == 1.0
+    assert associations["bucket"]["method"] == "correlation_ratio"
+    assert associations["bucket"]["score"] > 0.8
+
+
+def test_time_like_columns_trigger_split_review_guidance():
+    df = pd.DataFrame({
+        "event_date": pd.date_range("2026-01-01", periods=30, freq="D"),
+        "value": list(range(30)),
+        "target": [0, 1] * 15,
+    })
+
+    result = run_target_intelligence(df, target_column="target")
+
+    assert result["split_guidance"]["strategy"] == "review_time_aware_split"
+    assert "event_date" in result["split_guidance"]["time_candidates"]
+
+
+def test_target_intelligence_warns_for_identifier_like_target():
+    df = pd.DataFrame({
+        "feature": list(range(20)),
+        "customer_id": [f"CUST-{index:04d}" for index in range(20)],
+    })
+
+    result = run_target_intelligence(df, target_column="customer_id")
+    warning_codes = {item["code"] for item in result["warnings"]}
+
+    assert "target.id_like" in warning_codes
+
+
+def test_public_analyze_includes_target_intelligence_even_in_quick_mode():
+    df = pd.DataFrame({
+        "age": list(range(20, 40)),
+        "plan": ["basic", "pro"] * 10,
+        "churn": [0, 1] * 10,
+    })
+
+    result = framevitals.analyze(df, target="churn", mode="quick")
+
+    intelligence = result["target_intelligence"]
+    assert intelligence["available"] is True
+    assert intelligence["target_column"] == "churn"
+    assert intelligence["task_type"] == "classification"
+    assert isinstance(intelligence["top_associations"], list)
+    assert "target_intelligence" in result["timings_ms"]

From 150b88367659034e672551d00e69db5a5bab83e1 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:17:49 +0530
Subject: [PATCH 035/439] feat: normalize target intelligence warnings as
 findings

---
 src/framevitals/findings.py | 124 +++++++++++++++++++++++++++---------
 1 file changed, 95 insertions(+), 29 deletions(-)

diff --git a/src/framevitals/findings.py b/src/framevitals/findings.py
index a54d990..ed1418f 100644
--- a/src/framevitals/findings.py
+++ b/src/framevitals/findings.py
@@ -1,14 +1,14 @@
 """Normalized findings for FrameVitals analysis results.
 
-The current pipeline already produces display-oriented signals with evidence and
-recommendations.  This module converts those signals into a stable, machine-
-readable finding shape without changing any analysis thresholds.
+FrameVitals diagnostics originate in several deterministic analysis layers. This
+module translates their already-computed warnings/signals into one stable,
+machine-readable finding shape without changing diagnostic thresholds.
 """
 
 from __future__ import annotations
 
 import re
-from typing import Any, Iterable
+from typing import Any, Iterable, Mapping
 
 
 _SEVERITY_MAP = {
@@ -45,14 +45,17 @@ def normalize_severity(value: Any) -> str:
     return _SEVERITY_MAP.get(text, text or "none")
 
 
-def _is_actionable_signal(signal: dict[str, Any]) -> bool:
-    """Return whether an existing display signal represents an issue/review.
+def _sort_findings(findings: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    findings.sort(
+        key=lambda item: (
+            _SEVERITY_ORDER.get(str(item.get("severity")), 99),
+            str(item.get("code", "")),
+        )
+    )
+    return findings
 
-    Positive informational signals such as "Temporal Data: Detected" remain in
-    ``signals`` but do not become findings.  ML readiness is included whenever
-    it is not fully Ready because its existing signal does not use the generic
-    ``Review`` status.
-    """
+
+def _is_actionable_signal(signal: dict[str, Any]) -> bool:
     status = str(signal.get("status", "")).strip().lower()
     name = str(signal.get("name", "")).strip().lower()
 
@@ -64,12 +67,7 @@ def _is_actionable_signal(signal: dict[str, Any]) -> bool:
 
 
 def findings_from_signals(signals: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
-    """Normalize existing pipeline signals into actionable findings.
-
-    No diagnostic logic is introduced here: the function only translates
-    already-computed signal status/evidence/recommendations into a consistent
-    schema for library consumers and report renderers.
-    """
+    """Normalize existing display signals into actionable findings."""
     findings: list[dict[str, Any]] = []
 
     for signal in signals:
@@ -77,27 +75,95 @@ def findings_from_signals(signals: Iterable[dict[str, Any]]) -> list[dict[str, A
             continue
 
         name = str(signal.get("name") or "Finding")
-        severity = normalize_severity(signal.get("severity"))
-        finding = {
+        findings.append({
             "code": f"signal.{_slug(name)}",
             "title": name,
-            "severity": severity,
+            "severity": normalize_severity(signal.get("severity")),
             "scope": "dataset",
             "status": signal.get("status"),
             "evidence": signal.get("evidence") or "",
             "recommendation": signal.get("recommendation") or "",
             "method": "signal_engine",
             "confidence": "deterministic",
-        }
-        findings.append(finding)
+        })
 
-    findings.sort(
-        key=lambda item: (
-            _SEVERITY_ORDER.get(str(item.get("severity")), 99),
-            str(item.get("code", "")),
-        )
-    )
-    return findings
+    return _sort_findings(findings)
+
+
+def findings_from_target_intelligence(
+    target_intelligence: Mapping[str, Any] | None,
+) -> list[dict[str, Any]]:
+    """Normalize target-quality/leakage warnings into standard findings."""
+    if not isinstance(target_intelligence, Mapping) or not target_intelligence.get("available"):
+        return []
+
+    target = str(target_intelligence.get("target_column") or "target")
+    warnings = target_intelligence.get("warnings", [])
+    if not isinstance(warnings, list):
+        return []
+
+    findings: list[dict[str, Any]] = []
+    for warning in warnings:
+        if not isinstance(warning, Mapping):
+            continue
+        code = str(warning.get("code") or "target.warning")
+        evidence = str(warning.get("message") or "Target review is recommended.")
+
+        if code == "target.id_like":
+            title = "Identifier-like target"
+            recommendation = (
+                f"Verify that '{target}' represents an outcome rather than a record identifier "
+                "before training a supervised model."
+            )
+        elif code == "target.high_missingness":
+            title = "High target missingness"
+            recommendation = (
+                "Resolve or explicitly exclude rows with missing target labels before model training."
+            )
+        elif code == "target.high_cardinality_classification":
+            title = "High-cardinality classification target"
+            recommendation = (
+                "Confirm that the target classes are intentional and have enough examples for reliable evaluation."
+            )
+        elif code.startswith("target.leakage."):
+            feature = code.removeprefix("target.leakage.")
+            title = f"Potential target leakage: {feature}"
+            recommendation = (
+                f"Review or remove '{feature}' before training if it contains information that would "
+                "not be available at prediction time."
+            )
+        else:
+            title = "Target review"
+            recommendation = "Review the selected target and its relationship to the input features."
+
+        findings.append({
+            "code": code,
+            "title": title,
+            "severity": normalize_severity(warning.get("severity")),
+            "scope": "target",
+            "status": "Review",
+            "evidence": evidence,
+            "recommendation": recommendation,
+            "method": "target_intelligence",
+            "confidence": "deterministic",
+        })
+
+    return _sort_findings(findings)
+
+
+def merge_findings(*groups: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Merge finding groups, de-duplicating by stable code."""
+    merged: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for group in groups:
+        for finding in group:
+            code = str(finding.get("code") or "")
+            if code and code in seen:
+                continue
+            if code:
+                seen.add(code)
+            merged.append(dict(finding))
+    return _sort_findings(merged)
 
 
 def recommendations_from_findings(

From 9284c1e049dc77bd10096bed9e75bdfe4c46e187 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:18:17 +0530
Subject: [PATCH 036/439] feat: merge target warnings into normalized findings

---
 src/framevitals/result.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/result.py b/src/framevitals/result.py
index 7902f63..144cf85 100644
--- a/src/framevitals/result.py
+++ b/src/framevitals/result.py
@@ -12,7 +12,12 @@
 from pathlib import Path
 from typing import Any
 
-from framevitals.findings import findings_from_signals, recommendations_from_findings
+from framevitals.findings import (
+    findings_from_signals,
+    findings_from_target_intelligence,
+    merge_findings,
+    recommendations_from_findings,
+)
 
 
 class ColumnResult(dict):
@@ -37,7 +42,10 @@ def __init__(self, *args, **kwargs) -> None:
             signals = self.get("signals", [])
             if not isinstance(signals, list):
                 signals = []
-            self["findings"] = findings_from_signals(signals)
+            self["findings"] = merge_findings(
+                findings_from_signals(signals),
+                findings_from_target_intelligence(self.get("target_intelligence")),
+            )
 
     def __getattr__(self, name: str) -> Any:
         try:

From 72c95dbfed4520253696f2b34cb35ff7d54100a8 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:18:38 +0530
Subject: [PATCH 037/439] test: ensure target warnings reach normalized
 findings

---
 tests/test_target_intelligence_v2.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/tests/test_target_intelligence_v2.py b/tests/test_target_intelligence_v2.py
index 806324e..54d36d6 100644
--- a/tests/test_target_intelligence_v2.py
+++ b/tests/test_target_intelligence_v2.py
@@ -92,3 +92,31 @@ def test_public_analyze_includes_target_intelligence_even_in_quick_mode():
     assert intelligence["task_type"] == "classification"
     assert isinstance(intelligence["top_associations"], list)
     assert "target_intelligence" in result["timings_ms"]
+
+
+def test_target_leakage_warning_is_normalized_into_public_findings():
+    target = [0, 1] * 10
+    df = pd.DataFrame({
+        "leaked_target": target,
+        "age": list(range(20, 40)),
+        "churn": target,
+    })
+
+    result = framevitals.analyze(df, target="churn", mode="quick")
+    target_findings = [
+        finding
+        for finding in result.findings
+        if finding["method"] == "target_intelligence"
+    ]
+
+    assert any(
+        finding["code"] == "target.leakage.leaked_target"
+        for finding in target_findings
+    )
+    leakage_finding = next(
+        finding
+        for finding in target_findings
+        if finding["code"] == "target.leakage.leaked_target"
+    )
+    assert leakage_finding["severity"] == "critical"
+    assert "prediction time" in leakage_finding["recommendation"]

From aef578053be08e12ec8c049b67f66f0260c8899e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:19:53 +0530
Subject: [PATCH 038/439] feat: add explicit cleaning plan API

---
 src/framevitals/cleaning_plan.py | 244 +++++++++++++++++++++++++++++++
 1 file changed, 244 insertions(+)
 create mode 100644 src/framevitals/cleaning_plan.py

diff --git a/src/framevitals/cleaning_plan.py b/src/framevitals/cleaning_plan.py
new file mode 100644
index 0000000..ffffd79
--- /dev/null
+++ b/src/framevitals/cleaning_plan.py
@@ -0,0 +1,244 @@
+"""Explicit, conservative cleaning plans for FrameVitals datasets.
+
+Planning never mutates user data. Applying a plan returns a copy by default and
+supports only the conservative operations already used by FrameVitals' internal
+cleaner: duplicate removal and missing-value imputation by median/mode.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Mapping
+
+import numpy as np
+import pandas as pd
+
+from framevitals.health_score import calculate_health_score
+from framevitals.profiler import build_profile
+
+
+CLEANING_PLAN_SCHEMA_VERSION = "1"
+
+
+def _python_scalar(value: Any) -> Any:
+    if isinstance(value, np.generic):
+        return value.item()
+    if isinstance(value, pd.Timestamp):
+        return value.isoformat()
+    return value
+
+
+class CleaningPlan(dict):
+    """Dict-compatible, inspectable cleaning plan."""
+
+    def __getattr__(self, name: str) -> Any:
+        try:
+            return self[name]
+        except KeyError as exc:
+            raise AttributeError(name) from exc
+
+    @property
+    def actions(self) -> list[dict[str, Any]]:
+        value = self.get("actions", [])
+        return value if isinstance(value, list) else []
+
+    def summary(self) -> dict[str, Any]:
+        risk_counts: dict[str, int] = {}
+        for action in self.actions:
+            risk = str(action.get("risk") or "unknown").lower()
+            risk_counts[risk] = risk_counts.get(risk, 0) + 1
+        return {
+            "action_count": len(self.actions),
+            "risk_counts": risk_counts,
+            "duplicates_to_remove": self.get("duplicates_to_remove", 0),
+            "missing_values_to_fill": self.get("missing_values_to_fill", 0),
+        }
+
+    def to_json(
+        self,
+        destination: str | Path | None = None,
+        *,
+        indent: int = 2,
+    ) -> str | Path:
+        rendered = json.dumps(dict(self), indent=indent, default=str)
+        if destination is None:
+            return rendered
+        path = Path(destination)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(rendered + "\n", encoding="utf-8")
+        return path
+
+    def apply(self, dataframe: pd.DataFrame, *, copy: bool = True) -> pd.DataFrame:
+        return apply_cleaning_plan(dataframe, self, copy=copy)
+
+    def simulate(
+        self,
+        dataframe: pd.DataFrame,
+        *,
+        before_profile: dict | None = None,
+        before_health: dict | None = None,
+    ) -> dict[str, Any]:
+        return simulate_cleaning_plan(
+            dataframe,
+            self,
+            before_profile=before_profile,
+            before_health=before_health,
+        )
+
+
+def infer_cleaning_plan(
+    dataframe: pd.DataFrame,
+    *,
+    profile: dict | None = None,
+) -> CleaningPlan:
+    """Infer the same conservative operations used by the internal cleaner."""
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError("dataframe must be a pandas DataFrame")
+    if dataframe.empty:
+        raise ValueError("Cannot infer a cleaning plan for an empty DataFrame.")
+
+    if profile is None:
+        profile = build_profile(dataframe)
+
+    duplicate_count = int(profile.get("duplicate_rows", dataframe.duplicated().sum()))
+    working = dataframe.drop_duplicates() if duplicate_count else dataframe
+    actions: list[dict[str, Any]] = []
+
+    if duplicate_count:
+        actions.append({
+            "id": "remove_duplicates",
+            "type": "remove_duplicates",
+            "column": None,
+            "strategy": "drop_duplicate_rows",
+            "affected": duplicate_count,
+            "risk": "low",
+            "description": f"Remove {duplicate_count} duplicate rows.",
+        })
+
+    missing_values_to_fill = 0
+    missing_counts = working.isna().sum()
+    for column, missing_value in missing_counts.items():
+        missing = int(missing_value)
+        if missing == 0:
+            continue
+
+        missing_values_to_fill += missing
+        if pd.api.types.is_numeric_dtype(working[column]):
+            fill_value = working[column].median()
+            strategy = "median"
+            action_type = "fill_numeric_missing"
+            description = (
+                f"Fill {missing} missing values in '{column}' using the median."
+            )
+        else:
+            mode = working[column].mode(dropna=True)
+            fill_value = mode.iloc[0] if len(mode) else "Unknown"
+            strategy = "mode" if len(mode) else "constant"
+            action_type = "fill_categorical_missing"
+            description = (
+                f"Fill {missing} missing values in '{column}' using "
+                f"{'the mode' if len(mode) else 'a fallback value'}."
+            )
+
+        actions.append({
+            "id": f"{action_type}:{column}",
+            "type": action_type,
+            "column": str(column),
+            "strategy": strategy,
+            "fill_value": _python_scalar(fill_value),
+            "affected": missing,
+            "risk": "medium",
+            "description": description,
+        })
+
+    return CleaningPlan({
+        "schema_version": CLEANING_PLAN_SCHEMA_VERSION,
+        "actions": actions,
+        "duplicates_to_remove": duplicate_count,
+        "missing_values_to_fill": missing_values_to_fill,
+    })
+
+
+def apply_cleaning_plan(
+    dataframe: pd.DataFrame,
+    plan: Mapping[str, Any],
+    *,
+    copy: bool = True,
+) -> pd.DataFrame:
+    """Apply a validated FrameVitals cleaning plan and return the result."""
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError("dataframe must be a pandas DataFrame")
+    if not isinstance(plan, Mapping):
+        raise TypeError("plan must be a cleaning-plan mapping")
+    if plan.get("schema_version") != CLEANING_PLAN_SCHEMA_VERSION:
+        raise ValueError(
+            "Unsupported cleaning plan schema version: "
+            f"{plan.get('schema_version')!r}"
+        )
+
+    actions = plan.get("actions", [])
+    if not isinstance(actions, list):
+        raise ValueError("Cleaning plan actions must be a list.")
+
+    cleaned = dataframe.copy() if copy else dataframe
+    for action in actions:
+        if not isinstance(action, Mapping):
+            raise ValueError("Each cleaning plan action must be an object.")
+
+        action_type = action.get("type")
+        if action_type == "remove_duplicates":
+            cleaned.drop_duplicates(inplace=True)
+            continue
+
+        if action_type in {"fill_numeric_missing", "fill_categorical_missing"}:
+            column = action.get("column")
+            if not isinstance(column, str) or column not in cleaned.columns:
+                raise ValueError(f"Cleaning plan column not found: {column!r}")
+            cleaned[column] = cleaned[column].fillna(action.get("fill_value"))
+            continue
+
+        raise ValueError(f"Unsupported cleaning action type: {action_type!r}")
+
+    return cleaned
+
+
+def simulate_cleaning_plan(
+    dataframe: pd.DataFrame,
+    plan: Mapping[str, Any],
+    *,
+    before_profile: dict | None = None,
+    before_health: dict | None = None,
+) -> dict[str, Any]:
+    """Apply a plan to a copy and report expected quality/shape changes."""
+    if before_profile is None:
+        before_profile = build_profile(dataframe)
+    if before_health is None:
+        before_health = calculate_health_score(dataframe, before_profile)
+
+    cleaned = apply_cleaning_plan(dataframe, plan, copy=True)
+    after_profile = build_profile(cleaned)
+    after_health = calculate_health_score(cleaned, after_profile)
+
+    before_shape = before_profile.get("shape", {})
+    after_shape = after_profile.get("shape", {})
+    before_score = before_health.get("overall_score")
+    after_score = after_health.get("overall_score")
+
+    health_delta = None
+    if isinstance(before_score, (int, float)) and isinstance(after_score, (int, float)):
+        health_delta = round(float(after_score) - float(before_score), 4)
+
+    return {
+        "before_shape": dict(before_shape),
+        "after_shape": dict(after_shape),
+        "rows_removed": int(before_shape.get("rows", len(dataframe)))
+        - int(after_shape.get("rows", len(cleaned))),
+        "missing_before": int(dataframe.isna().sum().sum()),
+        "missing_after": int(cleaned.isna().sum().sum()),
+        "duplicates_before": int(before_profile.get("duplicate_rows", 0)),
+        "duplicates_after": int(after_profile.get("duplicate_rows", 0)),
+        "before_health": before_health,
+        "after_health": after_health,
+        "health_delta": health_delta,
+    }

From ad2951535fc4f212e5bae5d03adc3f4068ba8ce3 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:20:21 +0530
Subject: [PATCH 039/439] fix: preserve in-memory datetime fill values

---
 src/framevitals/cleaning_plan.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/framevitals/cleaning_plan.py b/src/framevitals/cleaning_plan.py
index ffffd79..ee486f9 100644
--- a/src/framevitals/cleaning_plan.py
+++ b/src/framevitals/cleaning_plan.py
@@ -24,8 +24,6 @@
 def _python_scalar(value: Any) -> Any:
     if isinstance(value, np.generic):
         return value.item()
-    if isinstance(value, pd.Timestamp):
-        return value.isoformat()
     return value
 
 

From 304590eee279d762f1a007d287f041334ad9fe08 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:20:35 +0530
Subject: [PATCH 040/439] refactor: drive internal cleaner from reusable
 cleaning plans

---
 src/framevitals/cleaner.py | 93 ++++++++++++++++++--------------------
 1 file changed, 43 insertions(+), 50 deletions(-)

diff --git a/src/framevitals/cleaner.py b/src/framevitals/cleaner.py
index 352dfde..a16cead 100644
--- a/src/framevitals/cleaner.py
+++ b/src/framevitals/cleaner.py
@@ -2,6 +2,7 @@
 
 import pandas as pd
 
+from framevitals.cleaning_plan import apply_cleaning_plan, infer_cleaning_plan
 from framevitals.health_score import calculate_health_score
 from framevitals.profiler import build_profile
 from framevitals.security import sanitize_csv_value
@@ -9,6 +10,37 @@
 CLEANED_DIR = Path("cleaned")
 
 
+def _legacy_action_view(action: dict) -> dict:
+    """Keep the existing cleaner action payload stable for current callers."""
+    action_type = action.get("type")
+    affected = int(action.get("affected", 0))
+    column = action.get("column")
+
+    if action_type == "remove_duplicates":
+        return {
+            "action": "Remove duplicates",
+            "details": f"Removed {affected} duplicate rows.",
+            "risk": "Low",
+        }
+    if action_type == "fill_numeric_missing":
+        return {
+            "action": "Fill numeric missing values",
+            "details": f"Filled {affected} missing values in '{column}' using median.",
+            "risk": "Medium",
+        }
+    if action_type == "fill_categorical_missing":
+        return {
+            "action": "Fill categorical missing values",
+            "details": f"Filled {affected} missing values in '{column}' using mode.",
+            "risk": "Medium",
+        }
+    return {
+        "action": str(action_type or "Cleaning action"),
+        "details": str(action.get("description") or ""),
+        "risk": str(action.get("risk") or "Unknown").title(),
+    }
+
+
 def create_cleaned_dataset(
     dataset_id,
     df,
@@ -20,66 +52,26 @@ def create_cleaned_dataset(
 ):
     """Build a conservative cleaned copy and optionally persist it as CSV.
 
-    Library callers can set ``write_output=False`` to keep analysis free of
-    filesystem side effects. Application callers retain the historical
-    behavior by leaving ``write_output`` enabled.
-
-    ``before_profile`` and ``before_health`` are optional cached values. The
-    main pipeline supplies them to avoid profiling/scoring the original data a
-    second time; standalone callers can omit them with unchanged behavior.
+    The implementation now uses the public cleaning-plan primitives internally,
+    but preserves the historical ``actions``/health/count payload for backward
+    compatibility. The structured plan is returned additionally under ``plan``.
     """
-    cleaned = df.copy()
-    actions = []
-
     if before_profile is None:
         before_profile = build_profile(df)
     if before_health is None:
         before_health = calculate_health_score(df, before_profile)
 
-    cached_duplicates = before_profile.get("duplicate_rows")
-    duplicate_count = int(
-        cached_duplicates if cached_duplicates is not None else df.duplicated().sum()
-    )
+    plan = infer_cleaning_plan(df, profile=before_profile)
+    cleaned = apply_cleaning_plan(df, plan, copy=True)
+    actions = [_legacy_action_view(action) for action in plan.actions]
+
+    after_profile = build_profile(cleaned)
+    after_health = calculate_health_score(cleaned, after_profile)
     missing_before = sum(
         int(value)
         for value in before_profile.get("missing_counts", {}).values()
         if value is not None
     )
-
-    if duplicate_count:
-        cleaned = cleaned.drop_duplicates()
-        actions.append({
-            "action": "Remove duplicates",
-            "details": f"Removed {duplicate_count} duplicate rows.",
-            "risk": "Low",
-        })
-
-    missing_counts = cleaned.isna().sum()
-    for col, missing_value in missing_counts.items():
-        missing = int(missing_value)
-        if missing == 0:
-            continue
-
-        if pd.api.types.is_numeric_dtype(cleaned[col]):
-            value = cleaned[col].median()
-            cleaned[col] = cleaned[col].fillna(value)
-            actions.append({
-                "action": "Fill numeric missing values",
-                "details": f"Filled {missing} missing values in '{col}' using median.",
-                "risk": "Medium",
-            })
-        else:
-            mode = cleaned[col].mode(dropna=True)
-            value = mode.iloc[0] if len(mode) else "Unknown"
-            cleaned[col] = cleaned[col].fillna(value)
-            actions.append({
-                "action": "Fill categorical missing values",
-                "details": f"Filled {missing} missing values in '{col}' using mode.",
-                "risk": "Medium",
-            })
-
-    after_profile = build_profile(cleaned)
-    after_health = calculate_health_score(cleaned, after_profile)
     missing_after = sum(
         int(value)
         for value in after_profile.get("missing_counts", {}).values()
@@ -96,11 +88,12 @@ def create_cleaned_dataset(
 
     return {
         "actions": actions,
+        "plan": dict(plan),
         "before_health": before_health,
         "after_health": after_health,
         "output_path": str(output_path) if output_path is not None else None,
         "missing_before": int(missing_before),
         "missing_after": int(missing_after),
-        "duplicates_before": duplicate_count,
+        "duplicates_before": int(plan.get("duplicates_to_remove", 0)),
         "duplicates_after": int(after_profile.get("duplicate_rows", 0)),
     }

From 7d22866dc614602cbf03ffbeecc21682d34fe2c0 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:21:02 +0530
Subject: [PATCH 041/439] feat: expose cleaning plan and explicit clean APIs

---
 src/framevitals/api.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index e789dc5..694998c 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -8,6 +8,11 @@
 import pandas as pd
 
 from framevitals.analysis_selector import select_analyses
+from framevitals.cleaning_plan import (
+    CleaningPlan,
+    apply_cleaning_plan,
+    infer_cleaning_plan,
+)
 from framevitals.column_roles import infer_column_roles
 from framevitals.config import ConfigInput, resolve_config
 from framevitals.contracts import infer_contract as _infer_contract
@@ -158,6 +163,28 @@ def plan(
     })
 
 
+def plan_cleaning(data: DataInput) -> CleaningPlan:
+    """Infer a conservative cleaning plan without modifying the input data."""
+    dataframe, _ = _load_input(data, label="Dataset")
+    profile = build_profile(dataframe)
+    return infer_cleaning_plan(dataframe, profile=profile)
+
+
+def clean(
+    data: DataInput,
+    *,
+    plan: Mapping[str, Any] | None = None,
+) -> pd.DataFrame:
+    """Return an explicitly cleaned copy of a dataset.
+
+    If ``plan`` is omitted FrameVitals infers its conservative default plan.
+    The original DataFrame or source file is never modified in place.
+    """
+    dataframe, _ = _load_input(data, label="Dataset")
+    resolved_plan = plan if plan is not None else infer_cleaning_plan(dataframe)
+    return apply_cleaning_plan(dataframe, resolved_plan, copy=True)
+
+
 def compare(
     reference: DataInput,
     current: DataInput,

From c3ef4933138bd98b35fa819d18bfa32c1aba6e21 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:21:20 +0530
Subject: [PATCH 042/439] feat: expose explicit cleaning APIs

---
 src/framevitals/__init__.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index b3bd591..f3befc1 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -4,6 +4,7 @@
 
 from typing import Any
 
+from framevitals.cleaning_plan import CleaningPlan
 from framevitals.config import AnalysisConfig
 from framevitals.planning import AnalysisPlan
 from framevitals.result import AnalysisResult, ColumnResult
@@ -63,6 +64,20 @@ def plan(
     )
 
 
+def plan_cleaning(data: Any) -> CleaningPlan:
+    """Infer a conservative cleaning plan without changing the input data."""
+    from framevitals.api import plan_cleaning as _plan_cleaning
+
+    return _plan_cleaning(data)
+
+
+def clean(data: Any, *, plan: Any = None):
+    """Return an explicitly cleaned copy using a supplied or inferred plan."""
+    from framevitals.api import clean as _clean
+
+    return _clean(data, plan=plan)
+
+
 def compare(
     reference: Any,
     current: Any,
@@ -100,9 +115,12 @@ def validate(data: Any, contract: dict[str, Any]) -> dict[str, Any]:
     "AnalysisPlan",
     "AnalysisResult",
     "AnalysisSnapshot",
+    "CleaningPlan",
     "ColumnResult",
     "analyze",
     "plan",
+    "plan_cleaning",
+    "clean",
     "compare",
     "infer_contract",
     "validate",

From 9bac73ec211bebac77a3dc1315f604233d960730 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:21:53 +0530
Subject: [PATCH 043/439] feat: add safe cleaning plan CLI workflow

---
 src/framevitals/cli.py | 72 ++++++++++++++++++++++++++++--------------
 1 file changed, 48 insertions(+), 24 deletions(-)

diff --git a/src/framevitals/cli.py b/src/framevitals/cli.py
index fc62b2f..9d35f8c 100644
--- a/src/framevitals/cli.py
+++ b/src/framevitals/cli.py
@@ -81,11 +81,7 @@ def build_parser() -> argparse.ArgumentParser:
         "analyze",
         help="Analyze a tabular dataset.",
     )
-    analyze_parser.add_argument(
-        "file",
-        type=Path,
-        help="Path to the dataset.",
-    )
+    analyze_parser.add_argument("file", type=Path, help="Path to the dataset.")
     _add_runtime_arguments(analyze_parser)
     analyze_parser.add_argument(
         "--artifacts",
@@ -111,11 +107,7 @@ def build_parser() -> argparse.ArgumentParser:
         "plan",
         help="Preview applicable analyses without running heavy stages.",
     )
-    plan_parser.add_argument(
-        "file",
-        type=Path,
-        help="Path to the dataset.",
-    )
+    plan_parser.add_argument("file", type=Path, help="Path to the dataset.")
     _add_runtime_arguments(plan_parser)
     plan_parser.add_argument(
         "--format",
@@ -125,20 +117,32 @@ def build_parser() -> argparse.ArgumentParser:
     )
     _add_output_argument(plan_parser)
 
+    clean_parser = subparsers.add_parser(
+        "clean",
+        help="Inspect a conservative cleaning plan and optionally write cleaned CSV.",
+    )
+    clean_parser.add_argument("file", type=Path, help="Path to the dataset.")
+    clean_parser.add_argument(
+        "--output",
+        type=Path,
+        default=None,
+        help="Write the cleaned dataset to this CSV path. Omit for plan-only mode.",
+    )
+    clean_parser.add_argument(
+        "--plan-output",
+        type=Path,
+        default=None,
+        help="Optional path to write the inferred cleaning plan as JSON.",
+    )
+
     compare_parser = subparsers.add_parser(
         "compare",
         help="Compare reference and current datasets for drift.",
     )
     compare_parser.add_argument(
-        "reference",
-        type=Path,
-        help="Reference/baseline dataset path.",
-    )
-    compare_parser.add_argument(
-        "current",
-        type=Path,
-        help="Current dataset path.",
+        "reference", type=Path, help="Reference/baseline dataset path."
     )
+    compare_parser.add_argument("current", type=Path, help="Current dataset path.")
     compare_parser.add_argument(
         "--columns",
         default=None,
@@ -157,9 +161,7 @@ def build_parser() -> argparse.ArgumentParser:
         help="Infer a reusable data contract from a reference dataset.",
     )
     infer_contract_parser.add_argument(
-        "file",
-        type=Path,
-        help="Path to the reference dataset.",
+        "file", type=Path, help="Path to the reference dataset."
     )
     _add_output_argument(infer_contract_parser)
 
@@ -168,9 +170,7 @@ def build_parser() -> argparse.ArgumentParser:
         help="Validate a dataset against a JSON data contract.",
     )
     validate_parser.add_argument(
-        "file",
-        type=Path,
-        help="Path to the dataset to validate.",
+        "file", type=Path, help="Path to the dataset to validate."
     )
     validate_parser.add_argument(
         "--contract",
@@ -273,6 +273,30 @@ def main() -> int:
                 print(f"Plan JSON     {args.output}")
         return 0
 
+    if args.command == "clean":
+        from framevitals.api import clean, plan_cleaning
+        from framevitals.security import sanitize_csv_value
+
+        cleaning_plan = plan_cleaning(args.file)
+        if args.plan_output is not None:
+            cleaning_plan.to_json(args.plan_output)
+
+        payload = {
+            "plan": dict(cleaning_plan),
+            "summary": cleaning_plan.summary(),
+            "cleaned_output": str(args.output) if args.output is not None else None,
+        }
+
+        if args.output is not None:
+            if args.output.suffix.lower() != ".csv":
+                raise ValueError("The clean command currently writes CSV output only.")
+            cleaned = clean(args.file, plan=cleaning_plan)
+            args.output.parent.mkdir(parents=True, exist_ok=True)
+            cleaned.map(sanitize_csv_value).to_csv(args.output, index=False)
+
+        print(json.dumps(payload, indent=2, default=str))
+        return 0
+
     if args.command == "compare":
         from framevitals.api import compare
 

From 64247eaf359514f2c7807933fdedd635362d4e51 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:22:13 +0530
Subject: [PATCH 044/439] test: cover explicit cleaning plans and legacy parity

---
 tests/test_cleaning_plans.py | 159 +++++++++++++++++++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100644 tests/test_cleaning_plans.py

diff --git a/tests/test_cleaning_plans.py b/tests/test_cleaning_plans.py
new file mode 100644
index 0000000..d27a395
--- /dev/null
+++ b/tests/test_cleaning_plans.py
@@ -0,0 +1,159 @@
+import json
+
+import pandas as pd
+
+import framevitals
+from framevitals.cleaner import create_cleaned_dataset
+from framevitals.cleaning_plan import infer_cleaning_plan
+from framevitals.cli import main
+from framevitals.health_score import calculate_health_score
+from framevitals.profiler import build_profile
+
+
+def _dirty_frame() -> pd.DataFrame:
+    return pd.DataFrame({
+        "age": [20.0, None, 30.0, 30.0],
+        "city": ["Pune", "Mumbai", None, None],
+    })
+
+
+def test_cleaning_plan_is_explicit_and_does_not_mutate_input():
+    df = _dirty_frame()
+    original = df.copy(deep=True)
+
+    plan = infer_cleaning_plan(df)
+
+    pd.testing.assert_frame_equal(df, original)
+    assert plan["schema_version"] == "1"
+    assert plan.summary()["action_count"] == 2
+    assert plan["duplicates_to_remove"] == 1
+    assert plan["missing_values_to_fill"] == 2
+    assert [action["type"] for action in plan.actions] == [
+        "remove_duplicates",
+        "fill_numeric_missing",
+    ]
+
+
+def test_apply_cleaning_plan_returns_clean_copy():
+    df = _dirty_frame()
+    plan = infer_cleaning_plan(df)
+
+    cleaned = plan.apply(df)
+
+    assert cleaned is not df
+    assert len(cleaned) == 3
+    assert int(cleaned.isna().sum().sum()) == 0
+    assert cleaned.loc[1, "age"] == 25.0
+    assert cleaned.loc[2, "city"] == "Pune"
+    assert pd.isna(df.loc[1, "age"])
+
+
+def test_cleaning_simulation_reports_expected_improvement():
+    df = _dirty_frame()
+    profile = build_profile(df)
+    health = calculate_health_score(df, profile)
+    plan = infer_cleaning_plan(df, profile=profile)
+
+    simulation = plan.simulate(
+        df,
+        before_profile=profile,
+        before_health=health,
+    )
+
+    assert simulation["rows_removed"] == 1
+    assert simulation["missing_before"] == 3
+    assert simulation["missing_after"] == 0
+    assert simulation["duplicates_before"] == 1
+    assert simulation["duplicates_after"] == 0
+    assert simulation["health_delta"] >= 0
+
+
+def test_internal_cleaner_preserves_legacy_payload_and_adds_structured_plan():
+    df = _dirty_frame()
+    result = create_cleaned_dataset("test", df, write_output=False)
+
+    assert result["actions"] == [
+        {
+            "action": "Remove duplicates",
+            "details": "Removed 1 duplicate rows.",
+            "risk": "Low",
+        },
+        {
+            "action": "Fill numeric missing values",
+            "details": "Filled 1 missing values in 'age' using median.",
+            "risk": "Medium",
+        },
+    ]
+    assert result["missing_before"] == 3
+    assert result["missing_after"] == 0
+    assert result["duplicates_before"] == 1
+    assert result["duplicates_after"] == 0
+    assert result["plan"]["schema_version"] == "1"
+
+
+def test_public_cleaning_api_supports_dataframe_and_file(tmp_path):
+    df = _dirty_frame()
+    plan = framevitals.plan_cleaning(df)
+    cleaned = framevitals.clean(df, plan=plan)
+
+    assert isinstance(plan, framevitals.CleaningPlan)
+    assert len(cleaned) == 3
+    assert cleaned.isna().sum().sum() == 0
+
+    path = tmp_path / "dirty.csv"
+    df.to_csv(path, index=False)
+    file_plan = framevitals.plan_cleaning(path)
+    file_cleaned = framevitals.clean(path, plan=file_plan)
+
+    assert file_plan.summary()["action_count"] == 2
+    assert len(file_cleaned) == 3
+    assert file_cleaned.isna().sum().sum() == 0
+
+
+def test_datetime_mode_fill_stays_datetime_dtype():
+    df = pd.DataFrame({
+        "event_time": [
+            pd.Timestamp("2026-01-01"),
+            pd.NaT,
+            pd.Timestamp("2026-01-01"),
+        ],
+    })
+    plan = infer_cleaning_plan(df)
+    cleaned = plan.apply(df)
+
+    assert pd.api.types.is_datetime64_any_dtype(cleaned["event_time"])
+    assert cleaned["event_time"].isna().sum() == 0
+
+
+def test_cli_clean_is_plan_only_without_output_and_explicit_with_output(
+    tmp_path,
+    monkeypatch,
+    capsys,
+):
+    dataset = tmp_path / "dirty.csv"
+    cleaned_path = tmp_path / "cleaned.csv"
+    plan_path = tmp_path / "plan.json"
+    _dirty_frame().to_csv(dataset, index=False)
+
+    monkeypatch.setattr(
+        "sys.argv",
+        ["framevitals", "clean", str(dataset), "--plan-output", str(plan_path)],
+    )
+    assert main() == 0
+    plan_only_payload = json.loads(capsys.readouterr().out)
+    assert plan_only_payload["cleaned_output"] is None
+    assert not cleaned_path.exists()
+    assert plan_path.exists()
+
+    monkeypatch.setattr(
+        "sys.argv",
+        ["framevitals", "clean", str(dataset), "--output", str(cleaned_path)],
+    )
+    assert main() == 0
+    apply_payload = json.loads(capsys.readouterr().out)
+    assert apply_payload["cleaned_output"] == str(cleaned_path)
+    assert cleaned_path.exists()
+
+    cleaned = pd.read_csv(cleaned_path)
+    assert len(cleaned) == 3
+    assert cleaned.isna().sum().sum() == 0

From 6247da203ceffeef6563de54ace7484e3a979da7 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:23:01 +0530
Subject: [PATCH 045/439] test: correct cleaning plan parity expectations

---
 tests/test_cleaning_plans.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tests/test_cleaning_plans.py b/tests/test_cleaning_plans.py
index d27a395..2f3e13b 100644
--- a/tests/test_cleaning_plans.py
+++ b/tests/test_cleaning_plans.py
@@ -25,12 +25,13 @@ def test_cleaning_plan_is_explicit_and_does_not_mutate_input():
 
     pd.testing.assert_frame_equal(df, original)
     assert plan["schema_version"] == "1"
-    assert plan.summary()["action_count"] == 2
+    assert plan.summary()["action_count"] == 3
     assert plan["duplicates_to_remove"] == 1
     assert plan["missing_values_to_fill"] == 2
     assert [action["type"] for action in plan.actions] == [
         "remove_duplicates",
         "fill_numeric_missing",
+        "fill_categorical_missing",
     ]
 
 
@@ -44,7 +45,7 @@ def test_apply_cleaning_plan_returns_clean_copy():
     assert len(cleaned) == 3
     assert int(cleaned.isna().sum().sum()) == 0
     assert cleaned.loc[1, "age"] == 25.0
-    assert cleaned.loc[2, "city"] == "Pune"
+    assert cleaned.loc[2, "city"] == "Mumbai"
     assert pd.isna(df.loc[1, "age"])
 
 
@@ -83,6 +84,11 @@ def test_internal_cleaner_preserves_legacy_payload_and_adds_structured_plan():
             "details": "Filled 1 missing values in 'age' using median.",
             "risk": "Medium",
         },
+        {
+            "action": "Fill categorical missing values",
+            "details": "Filled 1 missing values in 'city' using mode.",
+            "risk": "Medium",
+        },
     ]
     assert result["missing_before"] == 3
     assert result["missing_after"] == 0
@@ -105,7 +111,7 @@ def test_public_cleaning_api_supports_dataframe_and_file(tmp_path):
     file_plan = framevitals.plan_cleaning(path)
     file_cleaned = framevitals.clean(path, plan=file_plan)
 
-    assert file_plan.summary()["action_count"] == 2
+    assert file_plan.summary()["action_count"] == 3
     assert len(file_cleaned) == 3
     assert file_cleaned.isna().sum().sum() == 0
 

From 5ac9d48fcd9c36d3313ac2e5b28060aa2466d011 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:31:33 +0530
Subject: [PATCH 046/439] fix: preserve categorical target proxy detection in
 august integration

---
 src/framevitals/target_leakage.py | 63 ++++++++++++++++++++++++++++++-
 1 file changed, 62 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/target_leakage.py b/src/framevitals/target_leakage.py
index a4f9829..ead8e83 100644
--- a/src/framevitals/target_leakage.py
+++ b/src/framevitals/target_leakage.py
@@ -32,13 +32,66 @@ def numeric_correlation(a, b):
         return None, overlap
 
 
-def classify_target_leakage_risk(feature, target, same_ratio, corr):
+def categorical_mapping_purity(feature, target, *, max_categories=20):
+    """Measure whether low-cardinality feature values determine target labels.
+
+    A value of 1.0 means that every observed feature category maps to exactly one
+    target class. High-cardinality columns are intentionally skipped because
+    identifiers can trivially memorize a target without representing a useful
+    categorical leakage pattern.
+    """
+    both_present = feature.notna() & target.notna()
+    overlap = int(both_present.sum())
+    if overlap < 10:
+        return None, overlap
+
+    feature_values = feature.loc[both_present]
+    target_values = target.loc[both_present]
+    feature_cardinality = int(feature_values.nunique(dropna=True))
+    target_cardinality = int(target_values.nunique(dropna=True))
+
+    if (
+        feature_cardinality < 2
+        or feature_cardinality > max_categories
+        or target_cardinality < 2
+        or target_cardinality > max_categories
+    ):
+        return None, overlap
+
+    table = pd.crosstab(feature_values, target_values)
+    if table.empty:
+        return None, overlap
+
+    correctly_determined = int(table.max(axis=1).sum())
+    purity = correctly_determined / max(int(table.to_numpy().sum()), 1)
+    return round(float(purity), 4), overlap
+
+
+def classify_target_leakage_risk(
+    feature,
+    target,
+    same_ratio,
+    corr,
+    mapping_purity=None,
+):
     lower_feature = feature.lower()
     lower_target = target.lower()
 
     if same_ratio is not None and same_ratio >= 0.98:
         return "Critical", "Feature values almost exactly match the target on non-missing rows."
 
+    if mapping_purity is not None and mapping_purity >= 0.995:
+        return (
+            "Critical",
+            "Feature categories almost perfectly determine the target labels.",
+        )
+
+    if mapping_purity is not None and mapping_purity >= 0.98:
+        return (
+            "High",
+            "Feature categories very strongly determine the target labels.",
+        )
+
     if corr is not None and abs(corr) >= 0.98:
         return "High", "Feature is almost perfectly correlated with the target."
 
@@ -73,11 +126,17 @@ def run_target_leakage_analysis(df, target_column):
         if pd.api.types.is_numeric_dtype(feature) and pd.api.types.is_numeric_dtype(target):
             corr, corr_overlap = numeric_correlation(feature, target)
 
+        mapping_purity = None
+        mapping_overlap = 0
+        if not pd.api.types.is_numeric_dtype(feature):
+            mapping_purity, mapping_overlap = categorical_mapping_purity(feature, target)
+
         risk, reason = classify_target_leakage_risk(
             feature=column,
             target=target_column,
             same_ratio=same_ratio,
             corr=corr,
+            mapping_purity=mapping_purity,
         )
 
         if risk in {"Critical", "High", "Medium"}:
@@ -91,6 +150,8 @@ def run_target_leakage_analysis(df, target_column):
                     "same_overlap": same_overlap,
                     "correlation": corr,
                     "correlation_overlap": corr_overlap,
+                    "mapping_purity": mapping_purity,
+                    "mapping_overlap": mapping_overlap,
                 }
             )
 

From 46d471eacdc7a32ff53e64d287f5f4c41b5e5a29 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:42:05 +0530
Subject: [PATCH 047/439] feat: strengthen data contracts and validation gates

---
 src/framevitals/contracts.py | 382 +++++++++++++++++++++++++++++------
 1 file changed, 320 insertions(+), 62 deletions(-)

diff --git a/src/framevitals/contracts.py b/src/framevitals/contracts.py
index 3bf617b..3f04264 100644
--- a/src/framevitals/contracts.py
+++ b/src/framevitals/contracts.py
@@ -1,7 +1,15 @@
-"""Inference and validation helpers for lightweight data contracts."""
+"""Inference and validation helpers for reusable data contracts.
+
+Contracts are intentionally lightweight JSON-compatible dictionaries. Version 2
+adds tolerant numeric bounds, optional columns, low-cardinality value sets,
+null-rate ceilings, uniqueness hints, and normalized validation severities while
+remaining able to validate version-1 contracts created by earlier FrameVitals
+builds.
+"""
 
 from __future__ import annotations
 
+from collections import Counter
 from collections.abc import Mapping
 from typing import Any
 
@@ -9,7 +17,8 @@
 import pandas as pd
 from pandas.api import types as pdt
 
-CONTRACT_VERSION = 1
+CONTRACT_VERSION = 2
+_SUPPORTED_CONTRACT_VERSIONS = {1, 2}
 _TYPE_FAMILIES = {
     "boolean",
     "datetime",
@@ -17,6 +26,7 @@
     "number",
     "string",
 }
+_SEVERITIES = {"ignore", "warning", "error"}
 
 
 def _require_contractible_columns(dataframe: pd.DataFrame) -> None:
@@ -44,48 +54,150 @@ def _type_family(series: pd.Series) -> str:
     return "string"
 
 
-def _numeric_bounds(series: pd.Series) -> dict[str, int | float]:
+def _json_scalar(value: Any) -> Any:
+    if isinstance(value, np.generic):
+        return value.item()
+    if isinstance(value, pd.Timestamp):
+        return value.isoformat()
+    return value
+
+
+def _numeric_bounds(
+    series: pd.Series,
+    *,
+    tolerance: float,
+    family: str,
+) -> dict[str, int | float]:
     values = pd.to_numeric(series, errors="coerce").dropna()
     values = values[np.isfinite(values)]
     if values.empty:
         return {}
 
-    minimum = values.min()
-    maximum = values.max()
+    observed_min = float(values.min())
+    observed_max = float(values.max())
+    span = observed_max - observed_min
+    if span <= 0:
+        span = max(abs(observed_min), 1.0)
+    margin = span * tolerance
+
+    minimum = observed_min - margin
+    maximum = observed_max + margin
+    if family == "integer":
+        minimum = int(np.floor(minimum))
+        maximum = int(np.ceil(maximum))
+
     return {
-        "minimum": minimum.item() if hasattr(minimum, "item") else minimum,
-        "maximum": maximum.item() if hasattr(maximum, "item") else maximum,
+        "minimum": _json_scalar(minimum),
+        "maximum": _json_scalar(maximum),
+        "observed_minimum": _json_scalar(observed_min),
+        "observed_maximum": _json_scalar(observed_max),
     }
 
 
-def infer_contract(dataframe: pd.DataFrame) -> dict[str, Any]:
+def _allowed_values(series: pd.Series, *, max_categories: int) -> list[Any] | None:
+    clean = series.dropna()
+    if clean.empty:
+        return None
+    unique = clean.unique()
+    if len(unique) > max_categories:
+        return None
+    values = [_json_scalar(value) for value in unique.tolist()]
+    return sorted(values, key=lambda value: str(value))
+
+
+def infer_contract(
+    dataframe: pd.DataFrame,
+    *,
+    numeric_tolerance: float = 0.05,
+    max_categories: int = 20,
+    null_fraction_tolerance: float = 0.05,
+    infer_unique: bool = True,
+    min_unique_rows: int = 20,
+    allow_extra_columns: bool = False,
+) -> dict[str, Any]:
     """Create a JSON-serializable contract from a reference dataset.
 
-    The inferred contract captures schema and basic quality constraints that
-    are stable enough to use as an automated data gate: required columns,
-    broad type families, nullability, and finite numeric bounds.
+    Inference is deliberately conservative. Numeric extrema are expanded by a
+    configurable tolerance so ordinary future observations do not fail merely
+    because they exceed a reference sample's exact minimum or maximum. Small
+    categorical domains are recorded as warning-level expectations. Fully
+    unique columns are marked unique only when enough rows exist to make that
+    inference meaningful.
     """
     _require_contractible_columns(dataframe)
 
+    if numeric_tolerance < 0:
+        raise ValueError("numeric_tolerance must be non-negative.")
+    if max_categories < 1:
+        raise ValueError("max_categories must be at least 1.")
+    if not 0 <= null_fraction_tolerance <= 1:
+        raise ValueError("null_fraction_tolerance must be between 0 and 1.")
+    if min_unique_rows < 2:
+        raise ValueError("min_unique_rows must be at least 2.")
+
     columns: dict[str, dict[str, Any]] = {}
     for name in dataframe.columns:
         series = dataframe[name]
         family = _type_family(series)
+        null_fraction = float(series.isna().mean()) if len(series) else 0.0
+        clean = series.dropna()
+
         specification: dict[str, Any] = {
             "type": family,
+            "required": True,
             "nullable": bool(series.isna().any()),
+            "max_null_fraction": round(
+                min(1.0, null_fraction + null_fraction_tolerance),
+                6,
+            ),
         }
+
         if family in {"integer", "number"}:
-            specification.update(_numeric_bounds(series))
+            specification.update(
+                _numeric_bounds(
+                    series,
+                    tolerance=numeric_tolerance,
+                    family=family,
+                )
+            )
+        elif family in {"string", "boolean"}:
+            allowed = _allowed_values(series, max_categories=max_categories)
+            if allowed is not None:
+                specification["allowed_values"] = allowed
+                specification["allowed_values_severity"] = "warning"
+
+        if (
+            infer_unique
+            and len(clean) >= min_unique_rows
+            and int(clean.nunique(dropna=True)) == len(clean)
+        ):
+            specification["unique"] = True
+
         columns[name] = specification
 
     return {
         "version": CONTRACT_VERSION,
-        "allow_extra_columns": False,
+        "allow_extra_columns": bool(allow_extra_columns),
+        "extra_columns_severity": "ignore" if allow_extra_columns else "error",
+        "inference": {
+            "numeric_tolerance": float(numeric_tolerance),
+            "max_categories": int(max_categories),
+            "null_fraction_tolerance": float(null_fraction_tolerance),
+            "infer_unique": bool(infer_unique),
+            "min_unique_rows": int(min_unique_rows),
+        },
         "columns": columns,
     }
 
 
+def _normalise_severity(value: Any, *, field: str) -> str:
+    severity = str(value).strip().lower()
+    if severity not in _SEVERITIES:
+        choices = ", ".join(sorted(_SEVERITIES))
+        raise ValueError(f"contract field '{field}' must be one of: {choices}.")
+    return severity
+
+
 def _normalise_contract(contract: Mapping[str, Any]) -> dict[str, Any]:
     if not isinstance(contract, Mapping):
         raise TypeError("contract must be a mapping produced by infer_contract().")
@@ -94,14 +206,20 @@ def _normalise_contract(contract: Mapping[str, Any]) -> dict[str, Any]:
     if not isinstance(columns, Mapping) or not columns:
         raise ValueError("contract must contain a non-empty 'columns' mapping.")
 
-    version = contract.get("version", CONTRACT_VERSION)
-    if version != CONTRACT_VERSION:
+    version = contract.get("version", 1)
+    if version not in _SUPPORTED_CONTRACT_VERSIONS:
         raise ValueError(f"Unsupported contract version: {version!r}.")
 
     allow_extra_columns = contract.get("allow_extra_columns", False)
     if not isinstance(allow_extra_columns, bool):
         raise ValueError("contract field 'allow_extra_columns' must be a boolean.")
 
+    default_extra_severity = "ignore" if allow_extra_columns else "error"
+    extra_columns_severity = _normalise_severity(
+        contract.get("extra_columns_severity", default_extra_severity),
+        field="extra_columns_severity",
+    )
+
     normalised_columns: dict[str, dict[str, Any]] = {}
     for name, specification in columns.items():
         if not isinstance(name, str):
@@ -114,14 +232,34 @@ def _normalise_contract(contract: Mapping[str, Any]) -> dict[str, Any]:
             choices = ", ".join(sorted(_TYPE_FAMILIES))
             raise ValueError(f"contract type for '{name}' must be one of: {choices}.")
 
+        required = specification.get("required", True)
         nullable = specification.get("nullable", True)
-        if not isinstance(nullable, bool):
-            raise ValueError(f"contract field 'nullable' for '{name}' must be a boolean.")
+        unique = specification.get("unique", False)
+        for field_name, field_value in {
+            "required": required,
+            "nullable": nullable,
+            "unique": unique,
+        }.items():
+            if not isinstance(field_value, bool):
+                raise ValueError(
+                    f"contract field '{field_name}' for '{name}' must be a boolean."
+                )
 
-        normalised = {
+        normalised: dict[str, Any] = {
             "type": family,
+            "required": required,
             "nullable": nullable,
+            "unique": unique,
         }
+
+        max_null_fraction = specification.get("max_null_fraction")
+        if max_null_fraction is not None:
+            if not isinstance(max_null_fraction, int | float) or not 0 <= max_null_fraction <= 1:
+                raise ValueError(
+                    f"contract field 'max_null_fraction' for '{name}' must be between 0 and 1."
+                )
+            normalised["max_null_fraction"] = float(max_null_fraction)
+
         for bound in ("minimum", "maximum"):
             value = specification.get(bound)
             if value is not None:
@@ -135,11 +273,29 @@ def _normalise_contract(contract: Mapping[str, Any]) -> dict[str, Any]:
         upper = normalised.get("maximum")
         if lower is not None and upper is not None and lower > upper:
             raise ValueError(f"contract minimum exceeds maximum for '{name}'.")
+
+        allowed_values = specification.get("allowed_values")
+        if allowed_values is not None:
+            if family not in {"string", "boolean", "integer", "number"}:
+                raise ValueError(
+                    f"'allowed_values' is not supported for column '{name}' of type '{family}'."
+                )
+            if not isinstance(allowed_values, list) or not allowed_values:
+                raise ValueError(
+                    f"contract field 'allowed_values' for '{name}' must be a non-empty list."
+                )
+            normalised["allowed_values"] = allowed_values
+            normalised["allowed_values_severity"] = _normalise_severity(
+                specification.get("allowed_values_severity", "error"),
+                field=f"allowed_values_severity.{name}",
+            )
+
         normalised_columns[name] = normalised
 
     return {
         "version": version,
         "allow_extra_columns": allow_extra_columns,
+        "extra_columns_severity": extra_columns_severity,
         "columns": normalised_columns,
     }
 
@@ -150,18 +306,46 @@ def _compatible_type(expected: str, actual: str) -> bool:
     return expected == actual
 
 
-def _finding(code: str, column: str, message: str, **details: Any) -> dict[str, Any]:
+def _finding(
+    code: str,
+    column: str,
+    message: str,
+    *,
+    severity: str = "error",
+    **details: Any,
+) -> dict[str, Any]:
     finding: dict[str, Any] = {
         "code": code,
         "column": column,
+        "severity": severity,
         "message": message,
     }
     finding.update(details)
     return finding
 
 
+def _append_finding(
+    errors: list[dict[str, Any]],
+    warnings: list[dict[str, Any]],
+    finding: dict[str, Any],
+) -> None:
+    severity = finding.get("severity", "error")
+    if severity == "ignore":
+        return
+    if severity == "warning":
+        warnings.append(finding)
+    else:
+        errors.append(finding)
+
+
 def validate_contract(dataframe: pd.DataFrame, contract: Mapping[str, Any]) -> dict[str, Any]:
-    """Validate a dataset against an inferred or explicit data contract."""
+    """Validate a dataset against an inferred or explicit data contract.
+
+    Validation is aggregated rather than fail-fast: every applicable violation
+    is collected so CI, notebooks, and users can fix a dataset in one pass.
+    ``valid`` remains ``True`` when only warnings are present; ``status``
+    distinguishes ``pass``, ``warn``, and ``fail``.
+    """
     _require_contractible_columns(dataframe)
     normalised = _normalise_contract(contract)
     specifications = normalised["columns"]
@@ -171,24 +355,29 @@ def validate_contract(dataframe: pd.DataFrame, contract: Mapping[str, Any]) -> d
     actual_columns = set(dataframe.columns)
     expected_columns = set(specifications)
     for name in sorted(expected_columns - actual_columns):
-        errors.append(
-            _finding(
-                "missing_column",
-                name,
-                f"Required column '{name}' is missing.",
-            )
-        )
-
-    if not normalised["allow_extra_columns"]:
-        for name in sorted(actual_columns - expected_columns):
+        specification = specifications[name]
+        if specification.get("required", True):
             errors.append(
                 _finding(
-                    "unexpected_column",
+                    "missing_column",
                     name,
-                    f"Column '{name}' is not defined by the contract.",
+                    f"Required column '{name}' is missing.",
                 )
             )
 
+    extra_severity = normalised["extra_columns_severity"]
+    for name in sorted(actual_columns - expected_columns):
+        _append_finding(
+            errors,
+            warnings,
+            _finding(
+                "unexpected_column",
+                name,
+                f"Column '{name}' is not defined by the contract.",
+                severity=extra_severity,
+            ),
+        )
+
     for name in sorted(expected_columns & actual_columns):
         specification = specifications[name]
         series = dataframe[name]
@@ -208,6 +397,7 @@ def validate_contract(dataframe: pd.DataFrame, contract: Mapping[str, Any]) -> d
             continue
 
         null_count = int(series.isna().sum())
+        null_fraction = float(series.isna().mean()) if len(series) else 0.0
         if not specification["nullable"] and null_count:
             errors.append(
                 _finding(
@@ -215,8 +405,62 @@ def validate_contract(dataframe: pd.DataFrame, contract: Mapping[str, Any]) -> d
                     name,
                     f"Column '{name}' does not allow null values ({null_count} found).",
                     null_count=null_count,
+                    null_fraction=round(null_fraction, 6),
                 )
             )
+        else:
+            max_null_fraction = specification.get("max_null_fraction")
+            if max_null_fraction is not None and null_fraction > max_null_fraction:
+                warnings.append(
+                    _finding(
+                        "null_fraction_violation",
+                        name,
+                        (
+                            f"Column '{name}' has {null_fraction:.1%} null values, above "
+                            f"its expected ceiling of {max_null_fraction:.1%}."
+                        ),
+                        severity="warning",
+                        null_fraction=round(null_fraction, 6),
+                        max_null_fraction=max_null_fraction,
+                    )
+                )
+
+        if specification.get("unique"):
+            clean = series.dropna()
+            duplicate_count = int(clean.duplicated().sum())
+            if duplicate_count:
+                errors.append(
+                    _finding(
+                        "uniqueness_violation",
+                        name,
+                        f"Column '{name}' must be unique ({duplicate_count} duplicate values found).",
+                        duplicate_count=duplicate_count,
+                    )
+                )
+
+        allowed_values = specification.get("allowed_values")
+        if allowed_values is not None:
+            allowed = set(allowed_values)
+            observed = series.dropna().unique().tolist()
+            unexpected_values = [
+                _json_scalar(value)
+                for value in observed
+                if _json_scalar(value) not in allowed
+            ]
+            if unexpected_values:
+                severity = specification.get("allowed_values_severity", "error")
+                _append_finding(
+                    errors,
+                    warnings,
+                    _finding(
+                        "allowed_values_violation",
+                        name,
+                        f"Column '{name}' contains values outside its expected domain.",
+                        severity=severity,
+                        unexpected_values=unexpected_values[:20],
+                        unexpected_count=len(unexpected_values),
+                    ),
+                )
 
         if expected_type not in {"integer", "number"}:
             continue
@@ -226,45 +470,54 @@ def validate_contract(dataframe: pd.DataFrame, contract: Mapping[str, Any]) -> d
         if values.empty:
             continue
 
-        observed_minimum = values.min()
-        observed_maximum = values.max()
-        observed_minimum = (
-            observed_minimum.item()
-            if hasattr(observed_minimum, "item")
-            else observed_minimum
-        )
-        observed_maximum = (
-            observed_maximum.item()
-            if hasattr(observed_maximum, "item")
-            else observed_maximum
-        )
+        observed_minimum = _json_scalar(values.min())
+        observed_maximum = _json_scalar(values.max())
 
         minimum = specification.get("minimum")
-        if minimum is not None and observed_minimum < minimum:
-            errors.append(
-                _finding(
-                    "minimum_violation",
-                    name,
-                    f"Column '{name}' contains values below its minimum of {minimum}.",
-                    minimum=minimum,
-                    observed_minimum=observed_minimum,
+        if minimum is not None:
+            below_count = int((values < minimum).sum())
+            if below_count:
+                errors.append(
+                    _finding(
+                        "minimum_violation",
+                        name,
+                        f"Column '{name}' contains {below_count} values below its minimum of {minimum}.",
+                        minimum=minimum,
+                        observed_minimum=observed_minimum,
+                        violation_count=below_count,
+                    )
                 )
-            )
 
         maximum = specification.get("maximum")
-        if maximum is not None and observed_maximum > maximum:
-            errors.append(
-                _finding(
-                    "maximum_violation",
-                    name,
-                    f"Column '{name}' contains values above its maximum of {maximum}.",
-                    maximum=maximum,
-                    observed_maximum=observed_maximum,
+        if maximum is not None:
+            above_count = int((values > maximum).sum())
+            if above_count:
+                errors.append(
+                    _finding(
+                        "maximum_violation",
+                        name,
+                        f"Column '{name}' contains {above_count} values above its maximum of {maximum}.",
+                        maximum=maximum,
+                        observed_maximum=observed_maximum,
+                        violation_count=above_count,
+                    )
                 )
-            )
+
+    all_findings = errors + warnings
+    code_counts = dict(Counter(item["code"] for item in all_findings))
+    failed_columns = sorted({item["column"] for item in errors})
+    warning_columns = sorted({item["column"] for item in warnings})
+
+    if errors:
+        status = "fail"
+    elif warnings:
+        status = "warn"
+    else:
+        status = "pass"
 
     return {
         "valid": not errors,
+        "status": status,
         "contract_version": normalised["version"],
         "data": {
             "rows": int(len(dataframe)),
@@ -274,7 +527,12 @@ def validate_contract(dataframe: pd.DataFrame, contract: Mapping[str, Any]) -> d
             "columns_checked": len(specifications),
             "errors": len(errors),
             "warnings": len(warnings),
+            "findings": len(all_findings),
+            "failed_columns": failed_columns,
+            "warning_columns": warning_columns,
+            "code_counts": code_counts,
         },
         "errors": errors,
         "warnings": warnings,
+        "findings": all_findings,
     }

From e9f387fba6a7541eba6da3c8dd84d977db30042d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:43:05 +0530
Subject: [PATCH 048/439] feat: expand drift analysis into a quality gate

---
 src/framevitals/drift_analysis.py | 511 +++++++++++++++++++++++-------
 1 file changed, 393 insertions(+), 118 deletions(-)

diff --git a/src/framevitals/drift_analysis.py b/src/framevitals/drift_analysis.py
index f5eecfb..b6660fc 100644
--- a/src/framevitals/drift_analysis.py
+++ b/src/framevitals/drift_analysis.py
@@ -1,47 +1,38 @@
-"""
-Drift / Compare Analysis (WS-7)
-================================
-Compare two dataframes (a "reference" and a "current") column by column and
-quantify how much each column's distribution has shifted.
-
-Tests:
-    Numeric columns:
-        - Population Stability Index (PSI), 10 quantile bins
-        - Kolmogorov-Smirnov two-sample test (ks_2samp)
-        - Mean shift in standard deviations (z-shift)
-    Categorical columns:
-        - PSI on category proportions
-        - Chi-square test of independence
-
-Severity buckets (PSI):
-    < 0.10  -> stable
-    < 0.25  -> minor
-    < 0.50  -> moderate
-    >= 0.50 -> severe
-
-Public entry points:
-    compare_datasets(df_a, df_b, columns=None) -> dict
-    split_by_date(df, date_column, ratio=0.5) -> tuple[DataFrame, DataFrame]
-
-Output is fully JSON-safe.
+"""Dataset drift and change analysis.
+
+FrameVitals compares a reference and current dataframe across schema, types,
+missingness, and value distributions. PSI remains available for compatibility,
+but the public verdict now combines multiple interpretable signals rather than
+letting one statistic decide the entire result.
 """
 
 from __future__ import annotations
 
 import math
 import warnings
+from collections import Counter
 from typing import Any
 
 import numpy as np
 import pandas as pd
 from scipy import stats
+from scipy.spatial.distance import jensenshannon
+
+
+_SEVERITY_ORDER = {
+    "unknown": -1,
+    "stable": 0,
+    "minor": 1,
+    "moderate": 2,
+    "severe": 3,
+}
 
 
 def _safe_float(value: Any, ndigits: int = 4) -> float | int | None:
     if value is None:
         return None
     try:
-        if isinstance(value, (np.integer,)):
+        if isinstance(value, np.integer):
             return int(value)
         if isinstance(value, (np.floating, float)):
             v = float(value)
@@ -67,13 +58,90 @@ def _classify_psi(psi: float | None) -> str:
     return "severe"
 
 
+def _classify_missingness(delta_percentage_points: float) -> str:
+    delta = abs(delta_percentage_points)
+    if delta < 2:
+        return "stable"
+    if delta < 5:
+        return "minor"
+    if delta < 15:
+        return "moderate"
+    return "severe"
+
+
+def _classify_numeric_distance(value: float | None) -> str:
+    if value is None:
+        return "unknown"
+    if value < 0.10:
+        return "stable"
+    if value < 0.25:
+        return "minor"
+    if value < 0.50:
+        return "moderate"
+    return "severe"
+
+
+def _classify_js_distance(value: float | None) -> str:
+    if value is None:
+        return "unknown"
+    if value < 0.05:
+        return "stable"
+    if value < 0.10:
+        return "minor"
+    if value < 0.25:
+        return "moderate"
+    return "severe"
+
+
+def _max_severity(*values: str) -> str:
+    usable = [value for value in values if value in _SEVERITY_ORDER]
+    if not usable:
+        return "unknown"
+    return max(usable, key=lambda value: _SEVERITY_ORDER[value])
+
+
+def severity_at_least(actual: str, threshold: str) -> bool:
+    """Return whether a drift severity meets or exceeds ``threshold``."""
+    if threshold not in {"minor", "moderate", "severe"}:
+        raise ValueError("threshold must be one of: minor, moderate, severe.")
+    return _SEVERITY_ORDER.get(actual, -1) >= _SEVERITY_ORDER[threshold]
+
+
+def _dtype_family(series: pd.Series) -> str:
+    if pd.api.types.is_bool_dtype(series):
+        return "boolean"
+    if pd.api.types.is_datetime64_any_dtype(series):
+        return "datetime"
+    if pd.api.types.is_numeric_dtype(series):
+        return "numeric"
+    if (
+        pd.api.types.is_object_dtype(series)
+        or pd.api.types.is_string_dtype(series.dtype)
+        or isinstance(series.dtype, pd.CategoricalDtype)
+    ):
+        return "categorical"
+    return "unsupported"
+
+
 def _shared_columns(df_a: pd.DataFrame, df_b: pd.DataFrame) -> list[str]:
     other_columns = set(df_b.columns)
     return [column for column in df_a.columns if column in other_columns]
 
 
+def _missingness_metrics(ref: pd.Series, cur: pd.Series) -> dict[str, Any]:
+    ref_pct = float(ref.isna().mean() * 100) if len(ref) else 0.0
+    cur_pct = float(cur.isna().mean() * 100) if len(cur) else 0.0
+    delta = cur_pct - ref_pct
+    return {
+        "ref_missing_percent": _safe_float(ref_pct),
+        "cur_missing_percent": _safe_float(cur_pct),
+        "missingness_delta_percentage_points": _safe_float(delta),
+        "missingness_severity": _classify_missingness(delta),
+    }
+
+
 def _psi_numeric(ref: np.ndarray, cur: np.ndarray, bins: int = 10) -> float | None:
-    """Population Stability Index using quantile bins from `ref`."""
+    """Population Stability Index using quantile bins from ``ref``."""
     ref = ref[np.isfinite(ref)]
     cur = cur[np.isfinite(cur)]
     if len(ref) < 10 or len(cur) < 10:
@@ -88,10 +156,8 @@ def _psi_numeric(ref: np.ndarray, cur: np.ndarray, bins: int = 10) -> float | No
 
     ref_counts, _ = np.histogram(ref, bins=edges)
     cur_counts, _ = np.histogram(cur, bins=edges)
-
     ref_props = ref_counts / max(ref_counts.sum(), 1)
     cur_props = cur_counts / max(cur_counts.sum(), 1)
-
     ref_props = np.where(ref_props == 0, 1e-6, ref_props)
     cur_props = np.where(cur_props == 0, 1e-6, cur_props)
 
@@ -99,20 +165,36 @@ def _psi_numeric(ref: np.ndarray, cur: np.ndarray, bins: int = 10) -> float | No
     return psi if math.isfinite(psi) else None
 
 
-def _numeric_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict:
-    # Force float64 — pd.to_numeric leaves boolean dtype alone, and newer
-    # numpy refuses to subtract bool arrays inside np.quantile.
+def _normalized_wasserstein(ref: np.ndarray, cur: np.ndarray) -> float | None:
+    if len(ref) < 2 or len(cur) < 2:
+        return None
+    distance = float(stats.wasserstein_distance(ref, cur))
+    q1, q3 = np.quantile(ref, [0.25, 0.75])
+    scale = float(q3 - q1)
+    if scale <= 0:
+        scale = float(np.std(ref, ddof=0))
+    if scale <= 0:
+        scale = max(abs(float(np.mean(ref))), 1.0)
+    value = distance / scale
+    return value if math.isfinite(value) else None
+
+
+def _numeric_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict[str, Any]:
+    missingness = _missingness_metrics(ref, cur)
     ref_arr = pd.to_numeric(ref, errors="coerce").astype("float64").to_numpy()
     cur_arr = pd.to_numeric(cur, errors="coerce").astype("float64").to_numpy()
     ref_arr = ref_arr[np.isfinite(ref_arr)]
     cur_arr = cur_arr[np.isfinite(cur_arr)]
 
     if len(ref_arr) < 10 or len(cur_arr) < 10:
+        severity = missingness["missingness_severity"]
         return {
             "column": name,
             "type": "numeric",
             "available": False,
             "reason": "n<10 in one side",
+            "drift_severity": severity,
+            **missingness,
         }
 
     psi = _psi_numeric(ref_arr, cur_arr)
@@ -133,22 +215,25 @@ def _numeric_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict:
     pooled_std = ref_std if ref_std > 0 else cur_std if cur_std > 0 else 1.0
     z_shift = float((cur_mean - ref_mean) / pooled_std) if pooled_std > 0 else None
 
-    severity = _classify_psi(psi)
-    if ks_p is not None and ks_p < 0.01 and severity == "stable":
-        severity = "minor"
-
-    edges = np.linspace(
-        min(ref_arr.min(), cur_arr.min()),
-        max(ref_arr.max(), cur_arr.max()),
-        21,
+    wasserstein = _normalized_wasserstein(ref_arr, cur_arr)
+    psi_severity = _classify_psi(psi)
+    wasserstein_severity = _classify_numeric_distance(wasserstein)
+    ks_severity = "minor" if ks_p is not None and ks_p < 0.01 else "stable"
+    severity = _max_severity(
+        psi_severity,
+        wasserstein_severity,
+        ks_severity,
+        missingness["missingness_severity"],
     )
+
+    combined_min = float(min(ref_arr.min(), cur_arr.min()))
+    combined_max = float(max(ref_arr.max(), cur_arr.max()))
+    if combined_min == combined_max:
+        combined_min -= 0.5
+        combined_max += 0.5
+    edges = np.linspace(combined_min, combined_max, 21)
     ref_hist, _ = np.histogram(ref_arr, bins=edges)
     cur_hist, _ = np.histogram(cur_arr, bins=edges)
-    histogram = {
-        "edges": [_safe_float(v) for v in edges.tolist()],
-        "ref": [int(v) for v in ref_hist.tolist()],
-        "cur": [int(v) for v in cur_hist.tolist()],
-    }
 
     return {
         "column": name,
@@ -157,7 +242,9 @@ def _numeric_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict:
         "n_ref": int(len(ref_arr)),
         "n_cur": int(len(cur_arr)),
         "psi": _safe_float(psi),
-        "psi_severity": severity,
+        "psi_severity": psi_severity,
+        "wasserstein_normalized": _safe_float(wasserstein),
+        "wasserstein_severity": wasserstein_severity,
         "ks_statistic": ks_stat,
         "ks_p_value": ks_p,
         "ks_significant": bool(ks_p is not None and ks_p < 0.01),
@@ -166,25 +253,30 @@ def _numeric_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict:
         "ref_std": _safe_float(ref_std),
         "cur_std": _safe_float(cur_std),
         "z_shift": _safe_float(z_shift),
-        "histogram": histogram,
+        "drift_severity": severity,
+        **missingness,
+        "histogram": {
+            "edges": [_safe_float(value) for value in edges.tolist()],
+            "ref": [int(value) for value in ref_hist.tolist()],
+            "cur": [int(value) for value in cur_hist.tolist()],
+        },
     }
 
 
 def _psi_categorical(
     ref: pd.Series,
     cur: pd.Series,
-) -> tuple[float | None, list[str], dict, pd.Series, pd.Series]:
-    """Calculate categorical PSI and keep counts for downstream chi-square use."""
+) -> tuple[float | None, list[str], dict[str, Any], pd.Series, pd.Series, np.ndarray, np.ndarray]:
     ref_counts = ref.astype(str).value_counts(dropna=False)
     cur_counts = cur.astype(str).value_counts(dropna=False)
 
-    categories = sorted(set(ref_counts.index) | set(cur_counts.index))
+    combined = ref_counts.add(cur_counts, fill_value=0).sort_values(ascending=False)
+    categories = [str(value) for value in combined.index.tolist()]
     if len(categories) < 2:
-        return None, categories, {}, ref_counts, cur_counts
+        return None, categories, {}, ref_counts, cur_counts, np.array([]), np.array([])
 
     ref_total = max(int(ref_counts.sum()), 1)
     cur_total = max(int(cur_counts.sum()), 1)
-
     ref_props = np.array([ref_counts.get(c, 0) / ref_total for c in categories], dtype=float)
     cur_props = np.array([cur_counts.get(c, 0) / cur_total for c in categories], dtype=float)
 
@@ -199,40 +291,59 @@ def _psi_categorical(
         "ref_props": [round(float(p), 4) for p in ref_props[:30]],
         "cur_props": [round(float(p), 4) for p in cur_props[:30]],
     }
-
     return (
         psi if math.isfinite(psi) else None,
         categories,
         distribution,
         ref_counts,
         cur_counts,
+        ref_props,
+        cur_props,
     )
 
 
-def _categorical_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict:
+def _categorical_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict[str, Any]:
+    missingness = _missingness_metrics(ref, cur)
     ref_clean = ref.dropna()
     cur_clean = cur.dropna()
 
     if len(ref_clean) < 10 or len(cur_clean) < 10:
+        severity = missingness["missingness_severity"]
         return {
             "column": name,
             "type": "categorical",
             "available": False,
             "reason": "n<10 in one side",
+            "drift_severity": severity,
+            **missingness,
         }
 
-    psi, categories, distribution, ref_counts, cur_counts = _psi_categorical(
-        ref_clean,
-        cur_clean,
-    )
+    (
+        psi,
+        categories,
+        distribution,
+        ref_counts,
+        cur_counts,
+        ref_props,
+        cur_props,
+    ) = _psi_categorical(ref_clean, cur_clean)
     if psi is None:
         return {
             "column": name,
             "type": "categorical",
             "available": False,
             "reason": "single category",
+            "drift_severity": missingness["missingness_severity"],
+            **missingness,
         }
 
+    js_distance = None
+    if len(ref_props) and len(cur_props):
+        try:
+            js_distance = float(jensenshannon(ref_props, cur_props, base=2))
+        except Exception:
+            js_distance = None
+
     chi2_stat, chi2_p, dof = None, None, None
     try:
         top_categories = categories[:30]
@@ -254,6 +365,16 @@ def _categorical_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict
     ref_unique_set = set(ref_unique)
     cur_unique_set = set(cur_unique)
 
+    psi_severity = _classify_psi(psi)
+    js_severity = _classify_js_distance(js_distance)
+    chi_severity = "minor" if chi2_p is not None and chi2_p < 0.01 else "stable"
+    severity = _max_severity(
+        psi_severity,
+        js_severity,
+        chi_severity,
+        missingness["missingness_severity"],
+    )
+
     return {
         "column": name,
         "type": "categorical",
@@ -261,76 +382,187 @@ def _categorical_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict
         "n_ref": int(len(ref_clean)),
         "n_cur": int(len(cur_clean)),
         "psi": _safe_float(psi),
-        "psi_severity": _classify_psi(psi),
+        "psi_severity": psi_severity,
+        "jensen_shannon_distance": _safe_float(js_distance),
+        "jensen_shannon_severity": js_severity,
         "chi2_statistic": _safe_float(chi2_stat),
         "chi2_p_value": _safe_float(chi2_p),
         "chi2_dof": int(dof) if dof is not None else None,
         "chi2_significant": bool(chi2_p is not None and chi2_p < 0.01),
         "n_categories_ref": int(len(ref_unique)),
         "n_categories_cur": int(len(cur_unique)),
-        "new_categories": [c for c in cur_unique if c not in ref_unique_set][:10],
-        "missing_categories": [c for c in ref_unique if c not in cur_unique_set][:10],
+        "new_categories": [value for value in cur_unique if value not in ref_unique_set][:10],
+        "missing_categories": [value for value in ref_unique if value not in cur_unique_set][:10],
+        "drift_severity": severity,
+        **missingness,
         "distribution": distribution,
     }
 
 
+def _datetime_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict[str, Any]:
+    missingness = _missingness_metrics(ref, cur)
+    ref_dt = pd.to_datetime(ref, errors="coerce", utc=True).dropna()
+    cur_dt = pd.to_datetime(cur, errors="coerce", utc=True).dropna()
+    if len(ref_dt) < 10 or len(cur_dt) < 10:
+        return {
+            "column": name,
+            "type": "datetime",
+            "available": False,
+            "reason": "n<10 in one side",
+            "drift_severity": missingness["missingness_severity"],
+            **missingness,
+        }
+
+    ref_days = ref_dt.astype("int64").to_numpy(dtype="float64") / 86_400_000_000_000
+    cur_days = cur_dt.astype("int64").to_numpy(dtype="float64") / 86_400_000_000_000
+    psi = _psi_numeric(ref_days, cur_days)
+    wasserstein = _normalized_wasserstein(ref_days, cur_days)
+    try:
+        ks_stat, ks_p = stats.ks_2samp(ref_days, cur_days)
+    except Exception:
+        ks_stat, ks_p = None, None
+
+    psi_severity = _classify_psi(psi)
+    wasserstein_severity = _classify_numeric_distance(wasserstein)
+    ks_severity = "minor" if ks_p is not None and ks_p < 0.01 else "stable"
+    severity = _max_severity(
+        psi_severity,
+        wasserstein_severity,
+        ks_severity,
+        missingness["missingness_severity"],
+    )
+
+    return {
+        "column": name,
+        "type": "datetime",
+        "available": True,
+        "n_ref": int(len(ref_dt)),
+        "n_cur": int(len(cur_dt)),
+        "psi": _safe_float(psi),
+        "psi_severity": psi_severity,
+        "wasserstein_normalized": _safe_float(wasserstein),
+        "wasserstein_severity": wasserstein_severity,
+        "ks_statistic": _safe_float(ks_stat),
+        "ks_p_value": _safe_float(ks_p),
+        "ks_significant": bool(ks_p is not None and ks_p < 0.01),
+        "ref_min": ref_dt.min().isoformat(),
+        "ref_max": ref_dt.max().isoformat(),
+        "cur_min": cur_dt.min().isoformat(),
+        "cur_max": cur_dt.max().isoformat(),
+        "drift_severity": severity,
+        **missingness,
+    }
+
+
+def _column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict[str, Any]:
+    ref_family = _dtype_family(ref)
+    cur_family = _dtype_family(cur)
+    if ref_family != cur_family:
+        return {
+            "column": name,
+            "type": "type_mismatch",
+            "available": False,
+            "reason": f"Type changed from {ref_family} to {cur_family}.",
+            "ref_type": ref_family,
+            "cur_type": cur_family,
+            "drift_severity": "severe",
+            **_missingness_metrics(ref, cur),
+        }
+    if ref_family == "numeric":
+        return _numeric_column_drift(name, ref, cur)
+    if ref_family in {"categorical", "boolean"}:
+        return _categorical_column_drift(name, ref, cur)
+    if ref_family == "datetime":
+        return _datetime_column_drift(name, ref, cur)
+    return {
+        "column": name,
+        "type": "unsupported",
+        "available": False,
+        "reason": f"Unsupported dtype: {ref.dtype}",
+        "drift_severity": "unknown",
+        **_missingness_metrics(ref, cur),
+    }
+
+
 def compare_datasets(
     df_ref: pd.DataFrame,
     df_cur: pd.DataFrame,
     columns: list[str] | None = None,
     max_columns: int = 30,
-) -> dict:
-    """Compare two dataframes column by column and return a drift report."""
-    shared = _shared_columns(df_ref, df_cur)
+) -> dict[str, Any]:
+    """Compare two dataframes and return a structured drift/change report."""
+    if max_columns < 1:
+        raise ValueError("max_columns must be at least 1.")
+
+    ref_columns = set(df_ref.columns)
+    cur_columns = set(df_cur.columns)
+    all_shared = _shared_columns(df_ref, df_cur)
+    requested_missing_ref: list[str] = []
+    requested_missing_cur: list[str] = []
+
+    shared = all_shared
     if columns:
-        requested = set(columns)
-        shared = [column for column in shared if column in requested]
+        requested = list(dict.fromkeys(columns))
+        requested_set = set(requested)
+        requested_missing_ref = [name for name in requested if name not in ref_columns]
+        requested_missing_cur = [name for name in requested if name not in cur_columns]
+        shared = [name for name in all_shared if name in requested_set]
+
+    total_selected = len(shared)
+    truncated = total_selected > max_columns
     shared = shared[:max_columns]
 
+    added_columns = sorted(cur_columns - ref_columns)
+    removed_columns = sorted(ref_columns - cur_columns)
+    dtype_changes = [
+        {
+            "column": name,
+            "reference_type": _dtype_family(df_ref[name]),
+            "current_type": _dtype_family(df_cur[name]),
+        }
+        for name in all_shared
+        if _dtype_family(df_ref[name]) != _dtype_family(df_cur[name])
+    ]
+
+    row_delta_pct = None
+    if len(df_ref):
+        row_delta_pct = (len(df_cur) - len(df_ref)) / len(df_ref) * 100
+
     if not shared:
+        schema_severity = "severe" if removed_columns or dtype_changes else "moderate" if added_columns else "stable"
         return {
             "available": False,
-            "reason": "No shared columns between the two datasets.",
+            "reason": "No shared selected columns between the two datasets.",
             "ref_shape": list(df_ref.shape),
             "cur_shape": list(df_cur.shape),
+            "schema": {
+                "added_columns": added_columns,
+                "removed_columns": removed_columns,
+                "dtype_changes": dtype_changes,
+                "requested_missing_in_reference": requested_missing_ref,
+                "requested_missing_in_current": requested_missing_cur,
+                "severity": schema_severity,
+            },
         }
 
-    column_results: list[dict] = []
-    for col in shared:
-        ref_series = df_ref[col]
-        cur_series = df_cur[col]
-
-        if pd.api.types.is_numeric_dtype(ref_series) and pd.api.types.is_numeric_dtype(
-            cur_series
-        ):
-            column_results.append(_numeric_column_drift(col, ref_series, cur_series))
-        elif (
-            pd.api.types.is_object_dtype(ref_series)
-            or pd.api.types.is_string_dtype(ref_series.dtype)
-            or isinstance(ref_series.dtype, pd.CategoricalDtype)
-            or pd.api.types.is_bool_dtype(ref_series)
-        ):
-            column_results.append(_categorical_column_drift(col, ref_series, cur_series))
-        else:
-            column_results.append(
-                {
-                    "column": col,
-                    "type": "unsupported",
-                    "available": False,
-                    "reason": f"Unsupported dtype: {ref_series.dtype}",
-                }
-            )
-
-    severity_counts = {
-        "stable": 0,
-        "minor": 0,
-        "moderate": 0,
-        "severe": 0,
-        "unknown": 0,
-    }
-    for entry in column_results:
-        sev = entry.get("psi_severity") if entry.get("available") else "unknown"
-        severity_counts[sev] = severity_counts.get(sev, 0) + 1
+    column_results = [_column_drift(name, df_ref[name], df_cur[name]) for name in shared]
+
+    severity_counts = Counter(
+        entry.get("drift_severity", "unknown") for entry in column_results
+    )
+    for label in _SEVERITY_ORDER:
+        severity_counts.setdefault(label, 0)
+
+    schema_severity = "stable"
+    if removed_columns or dtype_changes:
+        schema_severity = "severe"
+    elif added_columns:
+        schema_severity = "moderate"
+
+    column_severity = _max_severity(
+        *(entry.get("drift_severity", "unknown") for entry in column_results)
+    )
+    overall_verdict = _max_severity(column_severity, schema_severity)
 
     severity_rank = {
         "severe": 0,
@@ -340,35 +572,76 @@ def compare_datasets(
         "unknown": 4,
     }
 
-    def _rank(entry):
-        sev = entry.get("psi_severity") if entry.get("available") else "unknown"
+    def _rank(entry: dict[str, Any]) -> tuple[int, float]:
+        severity = entry.get("drift_severity", "unknown")
         psi = entry.get("psi")
-        return (severity_rank.get(sev, 9), -(psi if isinstance(psi, (int, float)) else 0))
+        return (
+            severity_rank.get(severity, 9),
+            -(psi if isinstance(psi, (int, float)) else 0),
+        )
 
     column_results.sort(key=_rank)
 
-    if severity_counts["severe"] > 0:
-        verdict = "severe"
-    elif severity_counts["moderate"] > 0:
-        verdict = "moderate"
-    elif severity_counts["minor"] > 0:
-        verdict = "minor"
+    gate_reasons: list[str] = []
+    if removed_columns:
+        gate_reasons.append(f"{len(removed_columns)} reference columns are missing from current data.")
+    if dtype_changes:
+        gate_reasons.append(f"{len(dtype_changes)} shared columns changed type family.")
+    if added_columns:
+        gate_reasons.append(f"{len(added_columns)} new columns appeared in current data.")
+    severe_columns = [
+        entry["column"] for entry in column_results if entry.get("drift_severity") == "severe"
+    ]
+    moderate_columns = [
+        entry["column"] for entry in column_results if entry.get("drift_severity") == "moderate"
+    ]
+    if severe_columns:
+        gate_reasons.append(f"Severe drift detected in: {', '.join(severe_columns[:10])}.")
+    elif moderate_columns:
+        gate_reasons.append(f"Moderate drift detected in: {', '.join(moderate_columns[:10])}.")
+
+    if overall_verdict == "severe":
+        gate_status = "fail"
+    elif overall_verdict in {"minor", "moderate"}:
+        gate_status = "warn"
     else:
-        verdict = "stable"
+        gate_status = "pass"
 
     return {
         "available": True,
         "ref_shape": list(df_ref.shape),
         "cur_shape": list(df_cur.shape),
+        "row_count_change_percent": _safe_float(row_delta_pct),
         "shared_columns": shared,
+        "selection": {
+            "total_shared_columns": len(all_shared),
+            "total_selected_columns": total_selected,
+            "max_columns": int(max_columns),
+            "truncated": truncated,
+            "requested_missing_in_reference": requested_missing_ref,
+            "requested_missing_in_current": requested_missing_cur,
+        },
+        "schema": {
+            "added_columns": added_columns,
+            "removed_columns": removed_columns,
+            "dtype_changes": dtype_changes,
+            "severity": schema_severity,
+        },
         "summary": {
             "n_columns_compared": len(column_results),
-            "severity_counts": severity_counts,
-            "overall_verdict": verdict,
+            "n_columns_available": sum(bool(entry.get("available")) for entry in column_results),
+            "severity_counts": dict(severity_counts),
+            "overall_verdict": overall_verdict,
+        },
+        "gate": {
+            "status": gate_status,
+            "severity": overall_verdict,
+            "reasons": gate_reasons,
         },
         "interpretation": (
-            "PSI < 0.10 stable · 0.10-0.25 minor · 0.25-0.50 moderate · ≥ 0.50 severe. "
-            "Numeric columns also report KS test; categorical columns report chi-square."
+            "Drift severity combines PSI with normalized Wasserstein distance for numeric/date columns, "
+            "Jensen-Shannon distance for categorical columns, statistical tests, missingness changes, "
+            "and structural schema/type changes."
         ),
         "columns": column_results,
     }
@@ -382,6 +655,8 @@ def split_by_date(
     """Split a dataframe chronologically into earlier and later partitions."""
     if date_column not in df.columns:
         raise ValueError(f"Column not found: {date_column}")
+    if not 0 < ratio < 1:
+        raise ValueError("ratio must be between 0 and 1.")
 
     parsed = pd.to_datetime(df[date_column], errors="coerce", format="mixed")
     if parsed.notna().mean() < 0.7:

From eabc39e388bd6bd3fad77b143cf1f2f91aa3ce8a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:43:36 +0530
Subject: [PATCH 049/439] feat: expose contract inference controls

---
 src/framevitals/api.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index 694998c..c96d8fc 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -191,8 +191,8 @@ def compare(
     *,
     columns: list[str] | None = None,
     max_columns: int = 30,
-) -> dict:
-    """Compare reference and current datasets for distribution drift."""
+) -> dict[str, Any]:
+    """Compare reference and current datasets for distribution and schema drift."""
     if max_columns < 1:
         raise ValueError("max_columns must be at least 1.")
 
@@ -210,10 +210,27 @@ def compare(
     return result
 
 
-def infer_contract(data: DataInput) -> dict[str, Any]:
+def infer_contract(
+    data: DataInput,
+    *,
+    numeric_tolerance: float = 0.05,
+    max_categories: int = 20,
+    null_fraction_tolerance: float = 0.05,
+    infer_unique: bool = True,
+    min_unique_rows: int = 20,
+    allow_extra_columns: bool = False,
+) -> dict[str, Any]:
     """Infer a JSON-serializable data contract from a reference dataset."""
     dataframe, source_name = _load_input(data, label="Reference")
-    contract = _infer_contract(dataframe)
+    contract = _infer_contract(
+        dataframe,
+        numeric_tolerance=numeric_tolerance,
+        max_categories=max_categories,
+        null_fraction_tolerance=null_fraction_tolerance,
+        infer_unique=infer_unique,
+        min_unique_rows=min_unique_rows,
+        allow_extra_columns=allow_extra_columns,
+    )
     contract["reference_name"] = source_name
     return contract
 

From b9301c593d7ab6e06e003565891b0dfa59a44c8e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:43:51 +0530
Subject: [PATCH 050/439] feat: expose configurable contract inference

---
 src/framevitals/__init__.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index f3befc1..31ee11c 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -96,11 +96,28 @@ def compare(
     )
 
 
-def infer_contract(data: Any) -> dict[str, Any]:
+def infer_contract(
+    data: Any,
+    *,
+    numeric_tolerance: float = 0.05,
+    max_categories: int = 20,
+    null_fraction_tolerance: float = 0.05,
+    infer_unique: bool = True,
+    min_unique_rows: int = 20,
+    allow_extra_columns: bool = False,
+) -> dict[str, Any]:
     """Infer a reusable data contract from a reference dataset."""
     from framevitals.api import infer_contract as _infer_contract
 
-    return _infer_contract(data)
+    return _infer_contract(
+        data,
+        numeric_tolerance=numeric_tolerance,
+        max_categories=max_categories,
+        null_fraction_tolerance=null_fraction_tolerance,
+        infer_unique=infer_unique,
+        min_unique_rows=min_unique_rows,
+        allow_extra_columns=allow_extra_columns,
+    )
 
 
 def validate(data: Any, contract: dict[str, Any]) -> dict[str, Any]:

From e465ab93d405c441d82d011749dda036e68afe53 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:44:37 +0530
Subject: [PATCH 051/439] feat: add CLI quality gates for drift and validation

---
 src/framevitals/cli.py | 170 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 155 insertions(+), 15 deletions(-)

diff --git a/src/framevitals/cli.py b/src/framevitals/cli.py
index 9d35f8c..4bcb803 100644
--- a/src/framevitals/cli.py
+++ b/src/framevitals/cli.py
@@ -64,7 +64,7 @@ def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog="framevitals",
         description=(
-            "Automated diagnostics, ML-readiness analysis, and drift "
+            "Automated diagnostics, ML-readiness analysis, validation, and drift "
             "comparison for tabular datasets."
         ),
     )
@@ -137,7 +137,7 @@ def build_parser() -> argparse.ArgumentParser:
 
     compare_parser = subparsers.add_parser(
         "compare",
-        help="Compare reference and current datasets for drift.",
+        help="Compare reference and current datasets for schema/distribution drift.",
     )
     compare_parser.add_argument(
         "reference", type=Path, help="Reference/baseline dataset path."
@@ -154,6 +154,18 @@ def build_parser() -> argparse.ArgumentParser:
         default=30,
         help="Maximum number of shared columns to compare.",
     )
+    compare_parser.add_argument(
+        "--format",
+        choices=["json", "terminal"],
+        default="json",
+        help="Stdout format. JSON remains the default for backward compatibility.",
+    )
+    compare_parser.add_argument(
+        "--fail-on",
+        choices=["minor", "moderate", "severe"],
+        default=None,
+        help="Return exit code 1 when drift reaches this severity. Disabled by default.",
+    )
     _add_output_argument(compare_parser)
 
     infer_contract_parser = subparsers.add_parser(
@@ -163,6 +175,41 @@ def build_parser() -> argparse.ArgumentParser:
     infer_contract_parser.add_argument(
         "file", type=Path, help="Path to the reference dataset."
     )
+    infer_contract_parser.add_argument(
+        "--numeric-tolerance",
+        type=float,
+        default=0.05,
+        help="Expand inferred numeric bounds by this fraction of the observed span.",
+    )
+    infer_contract_parser.add_argument(
+        "--max-categories",
+        type=int,
+        default=20,
+        help="Infer allowed-value expectations up to this cardinality.",
+    )
+    infer_contract_parser.add_argument(
+        "--null-fraction-tolerance",
+        type=float,
+        default=0.05,
+        help="Additional tolerated null fraction above the reference rate.",
+    )
+    infer_contract_parser.add_argument(
+        "--infer-unique",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Infer uniqueness constraints for sufficiently large fully unique columns.",
+    )
+    infer_contract_parser.add_argument(
+        "--min-unique-rows",
+        type=int,
+        default=20,
+        help="Minimum non-null rows required before inferring uniqueness.",
+    )
+    infer_contract_parser.add_argument(
+        "--allow-extra-columns",
+        action="store_true",
+        help="Allow columns not present in the reference contract.",
+    )
     _add_output_argument(infer_contract_parser)
 
     validate_parser = subparsers.add_parser(
@@ -178,6 +225,17 @@ def build_parser() -> argparse.ArgumentParser:
         required=True,
         help="Path to a JSON contract created by infer-contract.",
     )
+    validate_parser.add_argument(
+        "--format",
+        choices=["json", "terminal"],
+        default="json",
+        help="Stdout format. JSON remains the default for backward compatibility.",
+    )
+    validate_parser.add_argument(
+        "--fail-on-warn",
+        action="store_true",
+        help="Return exit code 1 for warning-only validation results.",
+    )
     _add_output_argument(validate_parser)
 
     config_parser = subparsers.add_parser(
@@ -200,12 +258,68 @@ def build_parser() -> argparse.ArgumentParser:
     return parser
 
 
+def _write_json(payload: dict, output: Path) -> None:
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8")
+
+
 def _emit(payload: dict, output: Path | None) -> None:
-    rendered = json.dumps(payload, indent=2, default=str)
     if output is not None:
-        output.parent.mkdir(parents=True, exist_ok=True)
-        output.write_text(rendered + "\n", encoding="utf-8")
-    print(rendered)
+        _write_json(payload, output)
+    print(json.dumps(payload, indent=2, default=str))
+
+
+def _render_validation(report: dict) -> str:
+    summary = report.get("summary", {})
+    lines = [
+        "FrameVitals validation",
+        f"Status          {str(report.get('status', 'unknown')).upper()}",
+        f"Columns checked {summary.get('columns_checked', 0)}",
+        f"Errors          {summary.get('errors', 0)}",
+        f"Warnings        {summary.get('warnings', 0)}",
+    ]
+    findings = report.get("findings", [])
+    if findings:
+        lines.append("")
+        lines.append("Top findings")
+        for finding in findings[:8]:
+            lines.append(
+                f"- [{str(finding.get('severity', 'error')).upper()}] "
+                f"{finding.get('column')}: {finding.get('message')}"
+            )
+    return "\n".join(lines)
+
+
+def _render_compare(report: dict) -> str:
+    if not report.get("available"):
+        return f"FrameVitals drift\nStatus          UNAVAILABLE\nReason          {report.get('reason')}"
+
+    summary = report.get("summary", {})
+    gate = report.get("gate", {})
+    schema = report.get("schema", {})
+    lines = [
+        "FrameVitals drift",
+        f"Gate            {str(gate.get('status', 'unknown')).upper()}",
+        f"Severity        {str(summary.get('overall_verdict', 'unknown')).upper()}",
+        f"Columns checked {summary.get('n_columns_compared', 0)}",
+        f"Added columns   {len(schema.get('added_columns', []))}",
+        f"Removed columns {len(schema.get('removed_columns', []))}",
+        f"Type changes    {len(schema.get('dtype_changes', []))}",
+    ]
+    columns = report.get("columns", [])
+    notable = [
+        entry
+        for entry in columns
+        if entry.get("drift_severity") in {"minor", "moderate", "severe"}
+    ]
+    if notable:
+        lines.append("")
+        lines.append("Top drift")
+        for entry in notable[:8]:
+            lines.append(
+                f"- [{str(entry.get('drift_severity')).upper()}] {entry.get('column')}"
+            )
+    return "\n".join(lines)
 
 
 def main() -> int:
@@ -260,11 +374,7 @@ def main() -> int:
             config=args.config,
         )
         if args.output is not None:
-            args.output.parent.mkdir(parents=True, exist_ok=True)
-            args.output.write_text(
-                json.dumps(dict(result), indent=2, default=str) + "\n",
-                encoding="utf-8",
-            )
+            _write_json(dict(result), args.output)
         if args.format == "json":
             print(json.dumps(dict(result), indent=2, default=str))
         else:
@@ -299,6 +409,7 @@ def main() -> int:
 
     if args.command == "compare":
         from framevitals.api import compare
+        from framevitals.drift_analysis import severity_at_least
 
         columns = None
         if args.columns:
@@ -314,13 +425,32 @@ def main() -> int:
             columns=columns,
             max_columns=args.max_columns,
         )
-        _emit(report, args.output)
+        if args.output is not None:
+            _write_json(report, args.output)
+        if args.format == "terminal":
+            print(_render_compare(report))
+        else:
+            print(json.dumps(report, indent=2, default=str))
+
+        if args.fail_on and severity_at_least(
+            report.get("gate", {}).get("severity", "unknown"),
+            args.fail_on,
+        ):
+            return 1
         return 0
 
     if args.command == "infer-contract":
         from framevitals.api import infer_contract
 
-        report = infer_contract(args.file)
+        report = infer_contract(
+            args.file,
+            numeric_tolerance=args.numeric_tolerance,
+            max_categories=args.max_categories,
+            null_fraction_tolerance=args.null_fraction_tolerance,
+            infer_unique=args.infer_unique,
+            min_unique_rows=args.min_unique_rows,
+            allow_extra_columns=args.allow_extra_columns,
+        )
         _emit(report, args.output)
         return 0
 
@@ -331,8 +461,18 @@ def main() -> int:
             args.file,
             _load_contract(args.contract),
         )
-        _emit(report, args.output)
-        return 0 if report["valid"] else 1
+        if args.output is not None:
+            _write_json(report, args.output)
+        if args.format == "terminal":
+            print(_render_validation(report))
+        else:
+            print(json.dumps(report, indent=2, default=str))
+
+        if report.get("status") == "fail":
+            return 2
+        if report.get("status") == "warn" and args.fail_on_warn:
+            return 1
+        return 0
 
     if args.command == "config":
         from framevitals.config import resolve_config

From 93e8b86371bf0ea8c6a42a073965f739a2d15da8 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:45:25 +0530
Subject: [PATCH 052/439] test: cover tolerant contracts and aggregated
 validation

---
 tests/test_contracts.py | 213 +++++++++++++++++++++++++++++++---------
 1 file changed, 165 insertions(+), 48 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 6d7873e..b925e4c 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -20,55 +20,117 @@ def _reference_dataframe() -> pd.DataFrame:
     })
 
 
-def test_infer_contract_captures_json_safe_schema_and_bounds():
+def test_infer_contract_v2_is_json_safe_and_uses_tolerant_bounds():
     contract = infer_contract(_reference_dataframe())
 
-    assert contract == {
-        "version": 1,
-        "allow_extra_columns": False,
-        "columns": {
-            "account_id": {
-                "type": "integer",
-                "nullable": False,
-                "minimum": 101,
-                "maximum": 103,
-            },
-            "balance": {
-                "type": "number",
-                "nullable": False,
-                "minimum": 10.5,
-                "maximum": 40.0,
-            },
-            "plan": {"type": "string", "nullable": False},
-            "joined_at": {"type": "datetime", "nullable": False},
-        },
-    }
+    assert contract["version"] == 2
+    assert contract["allow_extra_columns"] is False
+    assert contract["extra_columns_severity"] == "error"
+    assert contract["inference"]["numeric_tolerance"] == 0.05
+
+    account = contract["columns"]["account_id"]
+    balance = contract["columns"]["balance"]
+    plan = contract["columns"]["plan"]
+
+    assert account["minimum"] < 101
+    assert account["maximum"] > 103
+    assert balance["minimum"] < 10.5
+    assert balance["maximum"] > 40.0
+    assert plan["allowed_values"] == ["basic", "premium", "standard"]
+    assert plan["allowed_values_severity"] == "warning"
     assert json.loads(json.dumps(contract)) == contract
 
 
-def test_validate_contract_accepts_compatible_data_without_mutating_it():
-    contract = infer_contract(_reference_dataframe())
+def test_inferred_contract_accepts_normal_future_values_inside_tolerance():
+    contract = infer_contract(_reference_dataframe(), numeric_tolerance=0.10)
     candidate = pd.DataFrame({
-        "account_id": [101, 102],
-        "balance": [11, 30],
+        "account_id": [100, 104],
+        "balance": [9.0, 42.0],
         "plan": ["basic", "premium"],
         "joined_at": pd.to_datetime(["2026-01-04", "2026-01-05"]),
     })
-    original = candidate.copy(deep=True)
 
     result = validate_contract(candidate, contract)
 
     assert result["valid"] is True
-    assert result["summary"] == {
-        "columns_checked": 4,
-        "errors": 0,
-        "warnings": 0,
-    }
-    pd.testing.assert_frame_equal(candidate, original)
+    assert result["status"] == "pass"
+    assert result["summary"]["errors"] == 0
 
 
-def test_validate_contract_reports_schema_type_nullability_and_bound_errors():
+def test_allowed_value_drift_is_warning_not_failure_by_default():
     contract = infer_contract(_reference_dataframe())
+    candidate = _reference_dataframe().copy()
+    candidate.loc[0, "plan"] = "enterprise"
+
+    result = validate_contract(candidate, contract)
+
+    assert result["valid"] is True
+    assert result["status"] == "warn"
+    assert result["errors"] == []
+    assert result["warnings"][0]["code"] == "allowed_values_violation"
+    assert result["warnings"][0]["severity"] == "warning"
+
+
+def test_inferred_uniqueness_requires_enough_rows_and_detects_duplicates():
+    reference = pd.DataFrame({
+        "customer_id": [f"CUST-{index:03d}" for index in range(30)],
+        "value": list(range(30)),
+    })
+    contract = infer_contract(reference)
+
+    assert contract["columns"]["customer_id"]["unique"] is True
+
+    candidate = reference.copy()
+    candidate.loc[1, "customer_id"] = candidate.loc[0, "customer_id"]
+    result = validate_contract(candidate, contract)
+
+    assert result["valid"] is False
+    assert any(item["code"] == "uniqueness_violation" for item in result["errors"])
+
+
+def test_nullable_column_can_warn_when_null_rate_exceeds_expected_ceiling():
+    contract = {
+        "version": 2,
+        "columns": {
+            "value": {
+                "type": "number",
+                "required": True,
+                "nullable": True,
+                "max_null_fraction": 0.10,
+            },
+        },
+    }
+    candidate = pd.DataFrame({"value": [1.0, None, None, 4.0]})
+
+    result = validate_contract(candidate, contract)
+
+    assert result["valid"] is True
+    assert result["status"] == "warn"
+    assert result["warnings"][0]["code"] == "null_fraction_violation"
+
+
+def test_optional_contract_column_may_be_absent():
+    contract = {
+        "version": 2,
+        "columns": {
+            "required_value": {"type": "integer", "nullable": False},
+            "optional_note": {
+                "type": "string",
+                "required": False,
+                "nullable": True,
+            },
+        },
+    }
+    candidate = pd.DataFrame({"required_value": [1, 2, 3]})
+
+    result = validate_contract(candidate, contract)
+
+    assert result["valid"] is True
+    assert result["status"] == "pass"
+
+
+def test_validation_aggregates_schema_type_nullability_and_bound_errors():
+    contract = infer_contract(_reference_dataframe(), numeric_tolerance=0)
     candidate = pd.DataFrame({
         "balance": [5.0, 55.0],
         "plan": [1, 2],
@@ -77,31 +139,65 @@ def test_validate_contract_reports_schema_type_nullability_and_bound_errors():
     })
 
     result = validate_contract(candidate, contract)
+    codes = {finding["code"] for finding in result["errors"]}
 
     assert result["valid"] is False
-    assert {finding["code"] for finding in result["errors"]} == {
+    assert result["status"] == "fail"
+    assert {
         "missing_column",
         "unexpected_column",
         "incompatible_type",
         "minimum_violation",
         "maximum_violation",
+    }.issubset(codes)
+    assert result["summary"]["findings"] == len(result["findings"])
+    assert result["summary"]["failed_columns"]
+    assert result["summary"]["code_counts"]
+
+
+def test_version_1_contracts_remain_supported():
+    contract = {
+        "version": 1,
+        "columns": {
+            "age": {
+                "type": "integer",
+                "nullable": False,
+                "minimum": 18,
+                "maximum": 100,
+            },
+        },
     }
 
+    result = validate_contract(pd.DataFrame({"age": [15, 25]}), contract)
 
-def test_validate_contract_rejects_nulls_when_contract_disallows_them():
-    contract = infer_contract(_reference_dataframe())
-    candidate = _reference_dataframe()
-    candidate.loc[1, "plan"] = None
+    assert result["contract_version"] == 1
+    assert result["status"] == "fail"
+    assert any(item["code"] == "minimum_violation" for item in result["errors"])
 
-    result = validate_contract(candidate, contract)
 
-    assert result["valid"] is False
-    assert result["errors"] == [{
-        "code": "nullability_violation",
-        "column": "plan",
-        "message": "Column 'plan' does not allow null values (1 found).",
-        "null_count": 1,
-    }]
+def test_extra_columns_can_be_allowed_or_downgraded_to_warning():
+    candidate = pd.DataFrame({"age": [20], "new_col": [1]})
+
+    allowed = validate_contract(
+        candidate,
+        {
+            "version": 2,
+            "allow_extra_columns": True,
+            "columns": {"age": {"type": "integer", "nullable": False}},
+        },
+    )
+    warning = validate_contract(
+        candidate,
+        {
+            "version": 2,
+            "extra_columns_severity": "warning",
+            "columns": {"age": {"type": "integer", "nullable": False}},
+        },
+    )
+
+    assert allowed["status"] == "pass"
+    assert warning["status"] == "warn"
+    assert warning["warnings"][0]["code"] == "unexpected_column"
 
 
 def test_validate_contract_rejects_invalid_contract_definitions():
@@ -122,16 +218,37 @@ def test_validate_contract_rejects_invalid_contract_definitions():
             },
         )
 
+    with pytest.raises(ValueError, match="between 0 and 1"):
+        validate_contract(
+            _reference_dataframe(),
+            {
+                "version": 2,
+                "columns": {
+                    "balance": {
+                        "type": "number",
+                        "max_null_fraction": 1.5,
+                    },
+                },
+            },
+        )
+
 
-def test_public_contract_api_accepts_file_paths_and_preserves_source_name(tmp_path):
+def test_public_contract_api_accepts_file_paths_and_inference_controls(tmp_path):
     reference_path = tmp_path / "reference.csv"
     candidate_path = tmp_path / "candidate.csv"
     _reference_dataframe().to_csv(reference_path, index=False)
     _reference_dataframe().to_csv(candidate_path, index=False)
 
-    contract = framevitals.infer_contract(reference_path)
+    contract = framevitals.infer_contract(
+        reference_path,
+        numeric_tolerance=0.10,
+        max_categories=10,
+        allow_extra_columns=True,
+    )
     result = framevitals.validate(candidate_path, contract)
 
     assert contract["reference_name"] == "reference.csv"
+    assert contract["inference"]["numeric_tolerance"] == 0.10
+    assert contract["allow_extra_columns"] is True
     assert result["dataset_name"] == "candidate.csv"
     assert result["valid"] is True

From ec345d296fbb78796d4e8d82a88b7d4e7490a0b1 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:45:42 +0530
Subject: [PATCH 053/439] test: cover CLI contract and drift gate controls

---
 tests/test_cli.py | 70 +++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 67 insertions(+), 3 deletions(-)

diff --git a/tests/test_cli.py b/tests/test_cli.py
index 9bedb0b..2182b3c 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -77,7 +77,7 @@ def test_cli_config_and_preset_arguments():
     assert config_args.preset == "deep"
 
 
-def test_cli_compare_parser():
+def test_cli_compare_parser_supports_gate_controls():
     parser = build_parser()
 
     args = parser.parse_args([
@@ -88,6 +88,10 @@ def test_cli_compare_parser():
         "age,income",
         "--max-columns",
         "12",
+        "--format",
+        "terminal",
+        "--fail-on",
+        "moderate",
         "--output",
         "drift.json",
     ])
@@ -97,15 +101,25 @@ def test_cli_compare_parser():
     assert args.current.name == "production.csv"
     assert args.columns == "age,income"
     assert args.max_columns == 12
+    assert args.format == "terminal"
+    assert args.fail_on == "moderate"
     assert args.output.name == "drift.json"
 
 
-def test_cli_contract_parsers():
+def test_cli_contract_parsers_expose_inference_and_warning_controls():
     parser = build_parser()
 
     infer_args = parser.parse_args([
         "infer-contract",
         "reference.csv",
+        "--numeric-tolerance",
+        "0.1",
+        "--max-categories",
+        "15",
+        "--null-fraction-tolerance",
+        "0.08",
+        "--no-infer-unique",
+        "--allow-extra-columns",
         "--output",
         "contract.json",
     ])
@@ -114,20 +128,30 @@ def test_cli_contract_parsers():
         "candidate.csv",
         "--contract",
         "contract.json",
+        "--format",
+        "terminal",
+        "--fail-on-warn",
         "--output",
         "validation.json",
     ])
 
     assert infer_args.command == "infer-contract"
     assert infer_args.file.name == "reference.csv"
+    assert infer_args.numeric_tolerance == 0.1
+    assert infer_args.max_categories == 15
+    assert infer_args.null_fraction_tolerance == 0.08
+    assert infer_args.infer_unique is False
+    assert infer_args.allow_extra_columns is True
     assert infer_args.output.name == "contract.json"
     assert validate_args.command == "validate"
     assert validate_args.file.name == "candidate.csv"
     assert validate_args.contract.name == "contract.json"
+    assert validate_args.format == "terminal"
+    assert validate_args.fail_on_warn is True
     assert validate_args.output.name == "validation.json"
 
 
-def test_cli_validate_returns_nonzero_for_contract_errors(tmp_path, monkeypatch):
+def test_cli_validate_returns_two_for_contract_errors(tmp_path, monkeypatch):
     dataset = tmp_path / "candidate.csv"
     contract = tmp_path / "contract.json"
     dataset.write_text("age\n15\n", encoding="utf-8")
@@ -156,4 +180,44 @@ def test_cli_validate_returns_nonzero_for_contract_errors(tmp_path, monkeypatch)
         ],
     )
 
+    assert main() == 2
+
+
+def test_cli_validate_warning_exit_is_opt_in(tmp_path, monkeypatch, capsys):
+    dataset = tmp_path / "candidate.csv"
+    contract = tmp_path / "contract.json"
+    dataset.write_text("plan\nenterprise\n", encoding="utf-8")
+    contract.write_text(
+        json.dumps({
+            "version": 2,
+            "columns": {
+                "plan": {
+                    "type": "string",
+                    "nullable": False,
+                    "allowed_values": ["basic", "pro"],
+                    "allowed_values_severity": "warning",
+                },
+            },
+        }),
+        encoding="utf-8",
+    )
+
+    monkeypatch.setattr(
+        "sys.argv",
+        ["framevitals", "validate", str(dataset), "--contract", str(contract)],
+    )
+    assert main() == 0
+    capsys.readouterr()
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "validate",
+            str(dataset),
+            "--contract",
+            str(contract),
+            "--fail-on-warn",
+        ],
+    )
     assert main() == 1

From 8ab5c5196ed773c16241587586f18679aaeab0fe Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:46:18 +0530
Subject: [PATCH 054/439] ci: test develop/august on push

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 46dde21..45ba443 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -2,7 +2,7 @@ name: Tests
 
 on:
   push:
-    branches: [main, dev]
+    branches: [main, dev, "develop/august"]
   pull_request:
 
 jobs:

From 75087a8a7c247922dcc079be7d61ba4b03e52829 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:46:29 +0530
Subject: [PATCH 055/439] ci: build develop/august on push

---
 .github/workflows/package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index b45eae6..be0fda2 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -2,7 +2,7 @@ name: Package
 
 on:
   push:
-    branches: [main, dev]
+    branches: [main, dev, "develop/august"]
   pull_request:
 
 jobs:

From 2ce84b9518a5665438da7f45203667148b0ccc06 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:46:51 +0530
Subject: [PATCH 056/439] test: add drift and quality gate coverage

---
 tests/test_quality_gates.py | 158 ++++++++++++++++++++++++++++++++++++
 1 file changed, 158 insertions(+)
 create mode 100644 tests/test_quality_gates.py

diff --git a/tests/test_quality_gates.py b/tests/test_quality_gates.py
new file mode 100644
index 0000000..5e63698
--- /dev/null
+++ b/tests/test_quality_gates.py
@@ -0,0 +1,158 @@
+import pandas as pd
+
+from framevitals.cli import main
+from framevitals.drift_analysis import compare_datasets, severity_at_least
+
+
+def test_identical_datasets_produce_pass_gate():
+    frame = pd.DataFrame({
+        "value": list(range(30)),
+        "group": ["A", "B", "C"] * 10,
+        "event_time": pd.date_range("2026-01-01", periods=30, freq="D"),
+    })
+
+    result = compare_datasets(frame, frame.copy())
+
+    assert result["available"] is True
+    assert result["gate"]["status"] == "pass"
+    assert result["summary"]["overall_verdict"] == "stable"
+    assert result["schema"]["severity"] == "stable"
+
+
+def test_added_schema_column_warns_without_failing():
+    reference = pd.DataFrame({"value": list(range(30))})
+    current = reference.assign(new_feature=1)
+
+    result = compare_datasets(reference, current)
+
+    assert result["schema"]["added_columns"] == ["new_feature"]
+    assert result["schema"]["removed_columns"] == []
+    assert result["schema"]["severity"] == "moderate"
+    assert result["gate"]["status"] == "warn"
+
+
+def test_removed_column_or_type_change_is_fail_level():
+    reference = pd.DataFrame({
+        "value": list(range(30)),
+        "segment": ["A", "B"] * 15,
+    })
+    current = pd.DataFrame({
+        "value": [str(value) for value in range(30)],
+    })
+
+    result = compare_datasets(reference, current)
+
+    assert result["schema"]["removed_columns"] == ["segment"]
+    assert result["schema"]["dtype_changes"][0]["column"] == "value"
+    assert result["summary"]["overall_verdict"] == "severe"
+    assert result["gate"]["status"] == "fail"
+
+
+def test_missingness_change_contributes_to_column_severity():
+    reference = pd.DataFrame({"value": list(range(50))})
+    current_values = list(range(20)) + [None] * 30
+    current = pd.DataFrame({"value": current_values})
+
+    result = compare_datasets(reference, current)
+    column = result["columns"][0]
+
+    assert column["cur_missing_percent"] == 60.0
+    assert column["missingness_delta_percentage_points"] == 60.0
+    assert column["missingness_severity"] == "severe"
+    assert column["drift_severity"] == "severe"
+
+
+def test_categorical_drift_reports_jensen_shannon_distance():
+    reference = pd.DataFrame({
+        "city": ["Pune"] * 30 + ["Mumbai"] * 20,
+    })
+    current = pd.DataFrame({
+        "city": ["Pune"] * 5 + ["Mumbai"] * 5 + ["Delhi"] * 40,
+    })
+
+    result = compare_datasets(reference, current)
+    column = result["columns"][0]
+
+    assert column["jensen_shannon_distance"] is not None
+    assert column["jensen_shannon_distance"] > 0
+    assert "Delhi" in column["new_categories"]
+    assert column["drift_severity"] in {"moderate", "severe"}
+
+
+def test_datetime_drift_is_analyzed_as_datetime():
+    reference = pd.DataFrame({
+        "event_time": pd.date_range("2026-01-01", periods=30, freq="D"),
+    })
+    current = pd.DataFrame({
+        "event_time": pd.date_range("2026-06-01", periods=30, freq="D"),
+    })
+
+    result = compare_datasets(reference, current)
+    column = result["columns"][0]
+
+    assert column["type"] == "datetime"
+    assert column["available"] is True
+    assert column["wasserstein_normalized"] is not None
+    assert column["ref_min"].startswith("2026-01-01")
+    assert column["cur_min"].startswith("2026-06-01")
+
+
+def test_requested_column_diagnostics_and_truncation_are_explicit():
+    reference = pd.DataFrame({
+        "a": list(range(30)),
+        "b": list(range(30)),
+        "c": list(range(30)),
+    })
+    current = reference.copy()
+
+    result = compare_datasets(
+        reference,
+        current,
+        columns=["a", "b", "c", "missing"],
+        max_columns=2,
+    )
+
+    assert result["selection"]["truncated"] is True
+    assert result["selection"]["total_selected_columns"] == 3
+    assert result["selection"]["requested_missing_in_reference"] == ["missing"]
+    assert result["selection"]["requested_missing_in_current"] == ["missing"]
+    assert result["shared_columns"] == ["a", "b"]
+
+
+def test_severity_threshold_helper_is_ordered():
+    assert severity_at_least("severe", "moderate") is True
+    assert severity_at_least("moderate", "moderate") is True
+    assert severity_at_least("minor", "moderate") is False
+    assert severity_at_least("stable", "minor") is False
+
+
+def test_compare_cli_fail_on_is_opt_in(tmp_path, monkeypatch, capsys):
+    reference = tmp_path / "reference.csv"
+    current = tmp_path / "current.csv"
+    pd.DataFrame({"value": list(range(50))}).to_csv(reference, index=False)
+    pd.DataFrame({"value": list(range(100, 150))}).to_csv(current, index=False)
+
+    monkeypatch.setattr(
+        "sys.argv",
+        ["framevitals", "compare", str(reference), str(current)],
+    )
+    assert main() == 0
+    capsys.readouterr()
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "compare",
+            str(reference),
+            str(current),
+            "--fail-on",
+            "moderate",
+            "--format",
+            "terminal",
+        ],
+    )
+    assert main() == 1
+    output = capsys.readouterr().out
+    assert "FrameVitals drift" in output
+    assert "Severity" in output

From f30e7f1b6862efabd5e85b5aa0095f3b1c072ba9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:47:29 +0530
Subject: [PATCH 057/439] feat: add drift and validation result objects

---
 src/framevitals/quality_results.py | 131 +++++++++++++++++++++++++++++
 1 file changed, 131 insertions(+)
 create mode 100644 src/framevitals/quality_results.py

diff --git a/src/framevitals/quality_results.py b/src/framevitals/quality_results.py
new file mode 100644
index 0000000..8573977
--- /dev/null
+++ b/src/framevitals/quality_results.py
@@ -0,0 +1,131 @@
+"""Dict-compatible public results for validation and drift workflows."""
+
+from __future__ import annotations
+
+import json
+from copy import deepcopy
+from pathlib import Path
+from typing import Any
+
+
+class _QualityResult(dict):
+    """Small mapping-compatible base with export conveniences."""
+
+    def __getattr__(self, name: str) -> Any:
+        try:
+            return self[name]
+        except KeyError as exc:
+            raise AttributeError(name) from exc
+
+    def to_dict(self) -> dict[str, Any]:
+        return deepcopy(dict(self))
+
+    def to_json(
+        self,
+        destination: str | Path | None = None,
+        *,
+        indent: int = 2,
+    ) -> str | Path:
+        rendered = json.dumps(self.to_dict(), indent=indent, default=str)
+        if destination is None:
+            return rendered
+        path = Path(destination)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(rendered + "\n", encoding="utf-8")
+        return path
+
+
+class ValidationResult(_QualityResult):
+    """Backward-compatible mapping returned by :func:`framevitals.validate`."""
+
+    @property
+    def valid(self) -> bool:
+        return bool(self.get("valid", False))
+
+    @property
+    def status(self) -> str:
+        return str(self.get("status", "unknown"))
+
+    @property
+    def findings(self) -> list[dict[str, Any]]:
+        value = self.get("findings", [])
+        return value if isinstance(value, list) else []
+
+    def summary_text(self) -> str:
+        summary = self.get("summary", {})
+        if not isinstance(summary, dict):
+            summary = {}
+        lines = [
+            "FrameVitals validation",
+            f"Status          {self.status.upper()}",
+            f"Columns checked {summary.get('columns_checked', 0)}",
+            f"Errors          {summary.get('errors', 0)}",
+            f"Warnings        {summary.get('warnings', 0)}",
+        ]
+        if self.findings:
+            lines.extend(["", "Top findings"])
+            for finding in self.findings[:8]:
+                lines.append(
+                    f"- [{str(finding.get('severity', 'error')).upper()}] "
+                    f"{finding.get('column')}: {finding.get('message')}"
+                )
+        return "\n".join(lines)
+
+
+class DriftResult(_QualityResult):
+    """Backward-compatible mapping returned by :func:`framevitals.compare`."""
+
+    @property
+    def severity(self) -> str:
+        gate = self.get("gate", {})
+        if isinstance(gate, dict):
+            return str(gate.get("severity", "unknown"))
+        return "unknown"
+
+    @property
+    def status(self) -> str:
+        gate = self.get("gate", {})
+        if isinstance(gate, dict):
+            return str(gate.get("status", "unknown"))
+        return "unknown"
+
+    @property
+    def columns(self) -> list[dict[str, Any]]:
+        value = self.get("columns", [])
+        return value if isinstance(value, list) else []
+
+    def summary_text(self) -> str:
+        if not self.get("available"):
+            return (
+                "FrameVitals drift\n"
+                "Status          UNAVAILABLE\n"
+                f"Reason          {self.get('reason')}"
+            )
+
+        summary = self.get("summary", {})
+        schema = self.get("schema", {})
+        if not isinstance(summary, dict):
+            summary = {}
+        if not isinstance(schema, dict):
+            schema = {}
+        lines = [
+            "FrameVitals drift",
+            f"Gate            {self.status.upper()}",
+            f"Severity        {self.severity.upper()}",
+            f"Columns checked {summary.get('n_columns_compared', 0)}",
+            f"Added columns   {len(schema.get('added_columns', []))}",
+            f"Removed columns {len(schema.get('removed_columns', []))}",
+            f"Type changes    {len(schema.get('dtype_changes', []))}",
+        ]
+        notable = [
+            entry
+            for entry in self.columns
+            if entry.get("drift_severity") in {"minor", "moderate", "severe"}
+        ]
+        if notable:
+            lines.extend(["", "Top drift"])
+            for entry in notable[:8]:
+                lines.append(
+                    f"- [{str(entry.get('drift_severity')).upper()}] {entry.get('column')}"
+                )
+        return "\n".join(lines)

From 54f16d9ae5719546f569df57cd1d030ea52bba97 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:47:52 +0530
Subject: [PATCH 058/439] feat: return typed drift and validation results

---
 src/framevitals/api.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index c96d8fc..a0c2e56 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -23,6 +23,7 @@
 from framevitals.pipeline import run_full_analysis
 from framevitals.planning import AnalysisPlan
 from framevitals.profiler import build_profile
+from framevitals.quality_results import DriftResult, ValidationResult
 from framevitals.result import AnalysisResult
 
 DataInput = str | Path | pd.DataFrame
@@ -191,7 +192,7 @@ def compare(
     *,
     columns: list[str] | None = None,
     max_columns: int = 30,
-) -> dict[str, Any]:
+) -> DriftResult:
     """Compare reference and current datasets for distribution and schema drift."""
     if max_columns < 1:
         raise ValueError("max_columns must be at least 1.")
@@ -199,15 +200,15 @@ def compare(
     ref_df, ref_name = _load_input(reference, label="Reference")
     cur_df, cur_name = _load_input(current, label="Current")
 
-    result = compare_datasets(
+    payload = compare_datasets(
         ref_df,
         cur_df,
         columns=columns,
         max_columns=max_columns,
     )
-    result["reference_name"] = ref_name
-    result["current_name"] = cur_name
-    return result
+    payload["reference_name"] = ref_name
+    payload["current_name"] = cur_name
+    return DriftResult(payload)
 
 
 def infer_contract(
@@ -238,9 +239,9 @@ def infer_contract(
 def validate(
     data: DataInput,
     contract: Mapping[str, Any],
-) -> dict[str, Any]:
+) -> ValidationResult:
     """Validate a dataset against an inferred or explicit data contract."""
     dataframe, source_name = _load_input(data, label="Dataset")
-    result = validate_contract(dataframe, contract)
-    result["dataset_name"] = source_name
-    return result
+    payload = validate_contract(dataframe, contract)
+    payload["dataset_name"] = source_name
+    return ValidationResult(payload)

From d7013e795c29a634feb7efe5529dbd693cc3f3fd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:48:10 +0530
Subject: [PATCH 059/439] feat: export drift and validation result types

---
 src/framevitals/__init__.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 31ee11c..1d49efa 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -7,6 +7,7 @@
 from framevitals.cleaning_plan import CleaningPlan
 from framevitals.config import AnalysisConfig
 from framevitals.planning import AnalysisPlan
+from framevitals.quality_results import DriftResult, ValidationResult
 from framevitals.result import AnalysisResult, ColumnResult
 from framevitals.snapshots import (
     AnalysisSnapshot,
@@ -84,7 +85,7 @@ def compare(
     *,
     columns: list[str] | None = None,
     max_columns: int = 30,
-) -> dict[str, Any]:
+) -> DriftResult:
     """Compare two datasets and return a structured drift report."""
     from framevitals.api import compare as _compare
 
@@ -120,7 +121,7 @@ def infer_contract(
     )
 
 
-def validate(data: Any, contract: dict[str, Any]) -> dict[str, Any]:
+def validate(data: Any, contract: dict[str, Any]) -> ValidationResult:
     """Validate a dataset against an inferred or explicit data contract."""
     from framevitals.api import validate as _validate
 
@@ -134,6 +135,8 @@ def validate(data: Any, contract: dict[str, Any]) -> dict[str, Any]:
     "AnalysisSnapshot",
     "CleaningPlan",
     "ColumnResult",
+    "DriftResult",
+    "ValidationResult",
     "analyze",
     "plan",
     "plan_cleaning",

From b7542010566cd575b7c0428d822d5a32f38f5893 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:48:33 +0530
Subject: [PATCH 060/439] test: cover public drift and validation result
 objects

---
 tests/test_quality_gates.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/tests/test_quality_gates.py b/tests/test_quality_gates.py
index 5e63698..c71733b 100644
--- a/tests/test_quality_gates.py
+++ b/tests/test_quality_gates.py
@@ -1,5 +1,8 @@
+import json
+
 import pandas as pd
 
+import framevitals
 from framevitals.cli import main
 from framevitals.drift_analysis import compare_datasets, severity_at_least
 
@@ -126,6 +129,31 @@ def test_severity_threshold_helper_is_ordered():
     assert severity_at_least("stable", "minor") is False
 
 
+def test_public_quality_results_are_dict_compatible_and_exportable(tmp_path):
+    reference = pd.DataFrame({
+        "value": list(range(30)),
+        "group": ["A", "B", "C"] * 10,
+    })
+
+    drift = framevitals.compare(reference, reference.copy())
+    assert isinstance(drift, framevitals.DriftResult)
+    assert isinstance(drift, dict)
+    assert drift.status == "pass"
+    assert "FrameVitals drift" in drift.summary_text()
+
+    drift_path = tmp_path / "drift.json"
+    drift.to_json(drift_path)
+    assert json.loads(drift_path.read_text(encoding="utf-8"))["gate"]["status"] == "pass"
+
+    contract = framevitals.infer_contract(reference)
+    validation = framevitals.validate(reference.copy(), contract)
+    assert isinstance(validation, framevitals.ValidationResult)
+    assert isinstance(validation, dict)
+    assert validation.valid is True
+    assert validation.status == "pass"
+    assert "FrameVitals validation" in validation.summary_text()
+
+
 def test_compare_cli_fail_on_is_opt_in(tmp_path, monkeypatch, capsys):
     reference = tmp_path / "reference.csv"
     current = tmp_path / "current.csv"

From 3892fe3f58a086299fc459c01d332b04f59882d8 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:50:56 +0530
Subject: [PATCH 061/439] feat: add configurable execution module controls

---
 src/framevitals/config.py | 83 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 79 insertions(+), 4 deletions(-)

diff --git a/src/framevitals/config.py b/src/framevitals/config.py
index 847dbe5..fc8201c 100644
--- a/src/framevitals/config.py
+++ b/src/framevitals/config.py
@@ -1,8 +1,9 @@
 """Runtime configuration for FrameVitals analysis.
 
-Configuration deliberately starts with options the current pipeline can honor
-fully.  Future category/model/backend settings can extend the same schema
-without introducing a second configuration system.
+The configuration layer controls both analysis depth/resources and optional
+pipeline modules. Defaults preserve historical behaviour; users can explicitly
+disable expensive or irrelevant modules without changing the stable result
+shape or maintaining a second configuration system.
 """
 
 from __future__ import annotations
@@ -14,13 +15,30 @@
 
 
 VALID_MODES = {"quick", "standard", "deep", "research"}
+VALID_MODULES = {
+    "deep_statistics",
+    "anomaly_detection",
+    "time_series",
+    "text_profile",
+    "target_intelligence",
+    "modeling",
+    "explainability",
+    "cleaning",
+    "charts",
+    "ai",
+}
 
 PRESETS: dict[str, dict[str, Any]] = {
     "quick": {"mode": "quick", "workers": 2, "artifacts": False},
     "standard": {"mode": "standard", "workers": 4, "artifacts": False},
     "deep": {"mode": "deep", "workers": 4, "artifacts": False},
     "research": {"mode": "research", "workers": 4, "artifacts": False},
-    "ci": {"mode": "standard", "workers": 2, "artifacts": False},
+    "ci": {
+        "mode": "standard",
+        "workers": 2,
+        "artifacts": False,
+        "disabled_modules": ("modeling", "explainability", "charts", "ai"),
+    },
 }
 
 
@@ -32,6 +50,7 @@ class AnalysisConfig:
     target: str | None = None
     artifacts: bool = False
     workers: int = 4
+    disabled_modules: tuple[str, ...] = ()
 
     def __post_init__(self) -> None:
         if self.mode not in VALID_MODES:
@@ -42,9 +61,23 @@ def __post_init__(self) -> None:
         if self.workers < 1:
             raise ValueError("workers must be at least 1.")
 
+        modules = tuple(dict.fromkeys(self.disabled_modules))
+        unknown = sorted(set(modules) - VALID_MODULES)
+        if unknown:
+            raise ValueError(
+                "Unknown FrameVitals module(s): "
+                f"{', '.join(unknown)}. Choose from: {', '.join(sorted(VALID_MODULES))}"
+            )
+        object.__setattr__(self, "disabled_modules", modules)
+
     def to_dict(self) -> dict[str, Any]:
         return asdict(self)
 
+    def module_enabled(self, name: str) -> bool:
+        if name not in VALID_MODULES:
+            raise ValueError(f"Unknown FrameVitals module: {name}")
+        return name not in self.disabled_modules
+
 
 ConfigInput = AnalysisConfig | Mapping[str, Any] | str | Path | None
 
@@ -54,6 +87,11 @@ def available_presets() -> tuple[str, ...]:
     return tuple(PRESETS)
 
 
+def available_modules() -> tuple[str, ...]:
+    """Return configurable execution module names in deterministic order."""
+    return tuple(sorted(VALID_MODULES))
+
+
 def _read_toml(path: str | Path) -> dict[str, Any]:
     source = Path(path)
     if not source.exists():
@@ -72,15 +110,26 @@ def _read_toml(path: str | Path) -> dict[str, Any]:
     return payload
 
 
+def _coerce_disabled_modules(value: Any) -> tuple[str, ...]:
+    if value is None:
+        return ()
+    if isinstance(value, str) or not isinstance(value, (list, tuple, set)):
+        raise ValueError("disabled_modules must be a list/tuple of module names.")
+    return tuple(str(item) for item in value)
+
+
 def _extract_values(mapping: Mapping[str, Any]) -> tuple[str | None, dict[str, Any]]:
     """Extract supported values from nested or flat configuration mappings."""
     analysis = mapping.get("analysis", {})
     resources = mapping.get("resources", {})
+    modules = mapping.get("modules", {})
 
     if not isinstance(analysis, Mapping):
         raise ValueError("[analysis] must be a TOML table/object.")
     if not isinstance(resources, Mapping):
         raise ValueError("[resources] must be a TOML table/object.")
+    if not isinstance(modules, Mapping):
+        raise ValueError("[modules] must be a TOML table/object.")
 
     preset = analysis.get("preset", mapping.get("preset"))
     values: dict[str, Any] = {}
@@ -96,6 +145,25 @@ def _extract_values(mapping: Mapping[str, Any]) -> tuple[str | None, dict[str, A
     elif "workers" in mapping:
         values["workers"] = mapping["workers"]
 
+    raw_disabled = analysis.get(
+        "disabled_modules",
+        mapping.get("disabled_modules", ()),
+    )
+    disabled = list(_coerce_disabled_modules(raw_disabled))
+
+    for name, enabled in modules.items():
+        if name not in VALID_MODULES:
+            raise ValueError(f"Unknown FrameVitals module in [modules]: {name}")
+        if not isinstance(enabled, bool):
+            raise ValueError(f"[modules].{name} must be true or false.")
+        if enabled:
+            disabled = [item for item in disabled if item != name]
+        elif name not in disabled:
+            disabled.append(name)
+
+    if disabled:
+        values["disabled_modules"] = tuple(disabled)
+
     return str(preset) if preset is not None else None, values
 
 
@@ -118,6 +186,7 @@ def resolve_config(
     target: str | None = None,
     artifacts: bool | None = None,
     workers: int | None = None,
+    disabled_modules: list[str] | tuple[str, ...] | None = None,
 ) -> AnalysisConfig:
     """Resolve defaults, preset, config file/object, then explicit overrides.
 
@@ -153,6 +222,9 @@ def resolve_config(
         "target": target,
         "artifacts": artifacts,
         "workers": workers,
+        "disabled_modules": (
+            tuple(disabled_modules) if disabled_modules is not None else None
+        ),
     }
     values.update({key: value for key, value in explicit.items() if value is not None})
 
@@ -166,6 +238,9 @@ def resolve_config(
     if values["target"] is not None and not isinstance(values["target"], str):
         raise ValueError("target must be a column name string or null.")
 
+    values["disabled_modules"] = _coerce_disabled_modules(
+        values.get("disabled_modules", ())
+    )
     return AnalysisConfig(**values)
 
 

From 1b855bc810d149c75267ffa37c4115a6d10dabba Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:52:17 +0530
Subject: [PATCH 062/439] feat: make execution modules configurable

---
 src/framevitals/pipeline.py | 266 ++++++++++++++++++++++++++----------
 1 file changed, 197 insertions(+), 69 deletions(-)

diff --git a/src/framevitals/pipeline.py b/src/framevitals/pipeline.py
index 0546122..4215f34 100644
--- a/src/framevitals/pipeline.py
+++ b/src/framevitals/pipeline.py
@@ -1,8 +1,10 @@
 """FrameVitals analysis pipeline.
 
-Runs the full analytics stack in phases and parallelizes independent analyses
-where safe. Optional failures are converted into structured error payloads so a
-single diagnostic does not sink the whole report.
+Runs the analytics stack in phases and parallelizes independent analyses where
+safe. Optional failures are converted into structured error payloads so a
+single diagnostic does not sink the whole report. Expensive modules can be
+explicitly disabled through ``AnalysisConfig`` while defaults preserve the
+historical execution path.
 """
 
 from __future__ import annotations
@@ -21,6 +23,7 @@
 from framevitals.anomaly_ensemble import detect_anomalies_ensemble
 from framevitals.cleaner import create_cleaned_dataset
 from framevitals.column_roles import infer_column_roles, summarize_roles
+from framevitals.config import VALID_MODULES
 from framevitals.dataset_signals import detect_dataset_signals
 from framevitals.deep_statistics_v2 import run_deep_statistics_v2
 from framevitals.explainability import explain_winner
@@ -53,6 +56,21 @@ def _safe_call(name: str, fn: Callable[[], Any]) -> tuple[str, Any, float]:
     return name, value, elapsed_ms
 
 
+def _skipped_module(module: str, reason: str = "Disabled by configuration.") -> dict:
+    return {
+        "available": False,
+        "skipped": True,
+        "module": module,
+        "reason": reason,
+    }
+
+
+def _result_status(value: Any) -> str:
+    if isinstance(value, dict) and value.get("error"):
+        return "error"
+    return "ran"
+
+
 def run_full_analysis(
     dataset_id: str,
     file_path=None,
@@ -63,11 +81,25 @@ def run_full_analysis(
     parallel_workers: int = 4,
     dataframe: pd.DataFrame | None = None,
     write_artifacts: bool = True,
+    disabled_modules: tuple[str, ...] | list[str] | set[str] | None = None,
 ) -> dict:
     """Run the complete FrameVitals analysis pipeline."""
     overall_start = time.perf_counter()
     timings_ms: dict[str, Any] = {}
 
+    disabled = set(disabled_modules or ())
+    unknown_modules = sorted(disabled - VALID_MODULES)
+    if unknown_modules:
+        raise ValueError(
+            "Unknown disabled module(s): " + ", ".join(unknown_modules)
+        )
+    module_status: dict[str, str] = {
+        name: "pending" for name in sorted(VALID_MODULES)
+    }
+
+    def module_enabled(name: str) -> bool:
+        return name not in disabled
+
     # Phase 1: load and understand structure.
     t0 = time.perf_counter()
     if dataframe is not None:
@@ -109,9 +141,13 @@ def run_full_analysis(
         analysis_mode=analysis_mode,
         target_column=target_column,
     )
+    analysis_selection["execution_modules"] = {
+        "disabled": sorted(disabled),
+        "enabled": sorted(VALID_MODULES - disabled),
+    }
     timings_ms["analysis_selection"] = (time.perf_counter() - t0) * 1000
 
-    # Phase 2: quality scoring.
+    # Phase 2: quality scoring remains core and always available.
     t0 = time.perf_counter()
     health = calculate_health_score(df, profile)
     timings_ms["health"] = (time.perf_counter() - t0) * 1000
@@ -130,42 +166,60 @@ def run_full_analysis(
     time_series_analysis = None
     text_profile = None
 
-    if analysis_mode in {"standard", "deep", "research"}:
-        tasks: list[tuple[str, Callable[[], Any]]] = [
-            ("deep_statistics_v2", lambda: run_deep_statistics_v2(df)),
-            ("anomalies_v2", lambda: detect_anomalies_ensemble(df)),
-            (
-                "time_series",
-                lambda: detect_and_analyze_time_series(
-                    df,
-                    target_column=target_column,
-                ),
+    phase3_modules: list[tuple[str, str, Callable[[], Any]]] = [
+        ("deep_statistics", "deep_statistics_v2", lambda: run_deep_statistics_v2(df)),
+        ("anomaly_detection", "anomalies_v2", lambda: detect_anomalies_ensemble(df)),
+        (
+            "time_series",
+            "time_series",
+            lambda: detect_and_analyze_time_series(
+                df,
+                target_column=target_column,
             ),
-            ("text_profile", lambda: profile_text_columns(df)),
-        ]
+        ),
+        ("text_profile", "text_profile", lambda: profile_text_columns(df)),
+    ]
 
-        results: dict[str, Any] = {}
-        per_task_ms: dict[str, float] = {}
+    phase3_results: dict[str, Any] = {}
+    if analysis_mode in {"standard", "deep", "research"}:
+        tasks: list[tuple[str, str, Callable[[], Any]]] = []
+        for module, result_key, fn in phase3_modules:
+            if module_enabled(module):
+                tasks.append((module, result_key, fn))
+                module_status[module] = "scheduled"
+            else:
+                phase3_results[result_key] = _skipped_module(module)
+                module_status[module] = "disabled"
 
-        phase3_start = time.perf_counter()
-        with ThreadPoolExecutor(max_workers=parallel_workers) as executor:
-            futures = {
-                executor.submit(_safe_call, name, fn): name for name, fn in tasks
-            }
-            for future in as_completed(futures):
-                name, value, elapsed = future.result()
-                results[name] = value
-                per_task_ms[name] = elapsed
-
-        deep_statistics_v2 = results.get("deep_statistics_v2")
-        anomalies_v2 = results.get("anomalies_v2")
-        time_series_analysis = results.get("time_series")
-        text_profile = results.get("text_profile")
-
-        timings_ms["phase3_parallel_total"] = (
-            time.perf_counter() - phase3_start
-        ) * 1000
+        per_task_ms: dict[str, float] = {}
+        if tasks:
+            phase3_start = time.perf_counter()
+            with ThreadPoolExecutor(max_workers=parallel_workers) as executor:
+                futures = {
+                    executor.submit(_safe_call, result_key, fn): (module, result_key)
+                    for module, result_key, fn in tasks
+                }
+                for future in as_completed(futures):
+                    module, result_key = futures[future]
+                    name, value, elapsed = future.result()
+                    phase3_results[result_key] = value
+                    per_task_ms[name] = elapsed
+                    module_status[module] = _result_status(value)
+
+            timings_ms["phase3_parallel_total"] = (
+                time.perf_counter() - phase3_start
+            ) * 1000
+        else:
+            timings_ms["phase3_parallel_total"] = 0.0
         timings_ms["phase3_tasks"] = per_task_ms
+    else:
+        for module, _, _ in phase3_modules:
+            module_status[module] = "not_applicable"
+
+    deep_statistics_v2 = phase3_results.get("deep_statistics_v2")
+    anomalies_v2 = phase3_results.get("anomalies_v2")
+    time_series_analysis = phase3_results.get("time_series")
+    text_profile = phase3_results.get("text_profile")
 
     # Phase 4: target-aware diagnostics and ML chain.
     target_intelligence = None
@@ -173,30 +227,49 @@ def run_full_analysis(
     explainability = None
 
     if target_column:
-        t0 = time.perf_counter()
-        _, target_intelligence, _ = _safe_call(
-            "target_intelligence",
-            lambda: run_target_intelligence(
-                df,
-                target_column=target_column,
-                column_roles=column_roles,
-            ),
-        )
-        timings_ms["target_intelligence"] = (time.perf_counter() - t0) * 1000
+        if module_enabled("target_intelligence"):
+            t0 = time.perf_counter()
+            _, target_intelligence, _ = _safe_call(
+                "target_intelligence",
+                lambda: run_target_intelligence(
+                    df,
+                    target_column=target_column,
+                    column_roles=column_roles,
+                ),
+            )
+            timings_ms["target_intelligence"] = (time.perf_counter() - t0) * 1000
+            module_status["target_intelligence"] = _result_status(target_intelligence)
+        else:
+            target_intelligence = _skipped_module("target_intelligence")
+            timings_ms["target_intelligence"] = 0.0
+            module_status["target_intelligence"] = "disabled"
+    else:
+        module_status["target_intelligence"] = "not_applicable"
 
-    if target_column and analysis_mode in {"standard", "deep", "research"}:
-        t0 = time.perf_counter()
-        _, model_leaderboard, _ = _safe_call(
-            "model_leaderboard",
-            lambda: run_model_leaderboard(df, target_column=target_column),
-        )
-        timings_ms["model_leaderboard"] = (time.perf_counter() - t0) * 1000
+    modeling_applicable = target_column and analysis_mode in {"standard", "deep", "research"}
+    if modeling_applicable:
+        if module_enabled("modeling"):
+            t0 = time.perf_counter()
+            _, model_leaderboard, _ = _safe_call(
+                "model_leaderboard",
+                lambda: run_model_leaderboard(df, target_column=target_column),
+            )
+            timings_ms["model_leaderboard"] = (time.perf_counter() - t0) * 1000
+            module_status["modeling"] = _result_status(model_leaderboard)
+        else:
+            model_leaderboard = _skipped_module("modeling")
+            timings_ms["model_leaderboard"] = 0.0
+            module_status["modeling"] = "disabled"
+    else:
+        module_status["modeling"] = "not_applicable"
 
-        if (
-            isinstance(model_leaderboard, dict)
-            and model_leaderboard.get("available")
-            and model_leaderboard.get("winner")
-        ):
+    winner_available = (
+        isinstance(model_leaderboard, dict)
+        and model_leaderboard.get("available")
+        and model_leaderboard.get("winner")
+    )
+    if winner_available:
+        if module_enabled("explainability"):
             t0 = time.perf_counter()
             _, explainability, _ = _safe_call(
                 "explainability",
@@ -208,24 +281,57 @@ def run_full_analysis(
                 ),
             )
             timings_ms["explainability"] = (time.perf_counter() - t0) * 1000
+            module_status["explainability"] = _result_status(explainability)
+        else:
+            explainability = _skipped_module("explainability")
+            timings_ms["explainability"] = 0.0
+            module_status["explainability"] = "disabled"
+    elif not module_enabled("explainability"):
+        explainability = _skipped_module("explainability")
+        module_status["explainability"] = "disabled"
+    else:
+        module_status["explainability"] = "not_applicable"
 
     # Phase 5: signals, cleaning, and optional visual artifacts.
     t0 = time.perf_counter()
     signals = build_signals(profile, health, ml_readiness, advanced)
     timings_ms["signals"] = (time.perf_counter() - t0) * 1000
 
-    t0 = time.perf_counter()
-    cleaning = create_cleaned_dataset(
-        dataset_id,
-        df,
-        write_output=write_artifacts,
-        before_profile=profile,
-        before_health=health,
-    )
-    timings_ms["cleaning"] = (time.perf_counter() - t0) * 1000
+    if module_enabled("cleaning"):
+        t0 = time.perf_counter()
+        cleaning = create_cleaned_dataset(
+            dataset_id,
+            df,
+            write_output=write_artifacts,
+            before_profile=profile,
+            before_health=health,
+        )
+        timings_ms["cleaning"] = (time.perf_counter() - t0) * 1000
+        module_status["cleaning"] = "ran"
+    else:
+        missing_count = sum(
+            int(value)
+            for value in profile.get("missing_counts", {}).values()
+            if value is not None
+        )
+        duplicate_count = int(profile.get("duplicate_rows", 0) or 0)
+        cleaning = {
+            **_skipped_module("cleaning"),
+            "actions": [],
+            "before_health": health,
+            "after_health": health,
+            "output_path": None,
+            "missing_before": missing_count,
+            "missing_after": missing_count,
+            "duplicates_before": duplicate_count,
+            "duplicates_after": duplicate_count,
+        }
+        timings_ms["cleaning"] = 0.0
+        module_status["cleaning"] = "disabled"
 
     charts: list[dict] = []
-    if write_artifacts and analysis_mode in {"standard", "deep", "research"}:
+    charts_applicable = write_artifacts and analysis_mode in {"standard", "deep", "research"}
+    if charts_applicable and module_enabled("charts"):
         t0 = time.perf_counter()
         charts = generate_charts(
             dataset_id,
@@ -240,6 +346,13 @@ def run_full_analysis(
             deep_statistics_v2=deep_statistics_v2,
         )
         timings_ms["charts"] = (time.perf_counter() - t0) * 1000
+        module_status["charts"] = "ran"
+    elif not module_enabled("charts"):
+        timings_ms["charts"] = 0.0
+        module_status["charts"] = "disabled"
+    else:
+        timings_ms["charts"] = 0.0
+        module_status["charts"] = "not_applicable"
 
     # Phase 6: optional AI interpretation.
     ai_env = os.environ.get(
@@ -248,13 +361,22 @@ def run_full_analysis(
     )
     analyze_ai_default = ai_env.strip().lower() in {"1", "true", "yes"}
 
-    if skip_ai or not analyze_ai_default:
+    if not module_enabled("ai"):
+        ai_report = {
+            "source": "disabled",
+            "text": "AI report disabled by configuration.",
+            "deferred": False,
+        }
+        timings_ms["ai_report"] = 0.0
+        module_status["ai"] = "disabled"
+    elif skip_ai or not analyze_ai_default:
         ai_report = {
             "source": "deferred" if not skip_ai else "skipped",
             "text": "" if not skip_ai else "AI report skipped.",
             "deferred": not skip_ai,
         }
         timings_ms["ai_report"] = 0.0
+        module_status["ai"] = "deferred" if not skip_ai else "skipped_by_caller"
     else:
         t0 = time.perf_counter()
         try:
@@ -267,9 +389,11 @@ def run_full_analysis(
                 column_roles_summary=roles_summary,
                 dataset_signals=dataset_signals,
             )
+            module_status["ai"] = "ran"
         except Exception as exc:  # noqa: BLE001
             logger.exception("AI report generation failed")
             ai_report = {"source": f"error: {exc}", "text": str(exc)}
+            module_status["ai"] = "error"
         timings_ms["ai_report"] = (time.perf_counter() - t0) * 1000
 
     timings_ms["total"] = (time.perf_counter() - overall_start) * 1000
@@ -287,6 +411,10 @@ def run_full_analysis(
         "filename": original_filename,
         "analysis_mode": analysis_mode,
         "artifacts_enabled": write_artifacts,
+        "execution": {
+            "disabled_modules": sorted(disabled),
+            "module_status": module_status,
+        },
         "profile": profile,
         "column_roles": column_roles,
         "roles_summary": roles_summary,

From c3bc4e8af4bfd43c65c4563fea788b8d4ec96050 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:53:25 +0530
Subject: [PATCH 063/439] feat: wire module controls through public analysis
 API

---
 src/framevitals/api.py | 51 ++++++++++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index a0c2e56..4fea9d9 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -65,6 +65,7 @@ def analyze(
     workers: int | None = None,
     preset: str | None = None,
     config: ConfigInput = None,
+    disabled_modules: list[str] | tuple[str, ...] | None = None,
 ) -> AnalysisResult:
     """Analyze a tabular dataset with FrameVitals."""
     resolved = resolve_config(
@@ -74,33 +75,34 @@ def analyze(
         target=target,
         artifacts=artifacts,
         workers=workers,
+        disabled_modules=disabled_modules,
     )
     dataset_id = f"fv_{uuid4().hex[:12]}"
 
+    pipeline_kwargs = {
+        "dataset_id": dataset_id,
+        "original_filename": "<dataframe>",
+        "analysis_mode": resolved.mode,
+        "target_column": resolved.target,
+        "parallel_workers": resolved.workers,
+        "skip_ai": True,
+        "write_artifacts": resolved.artifacts,
+        "disabled_modules": resolved.disabled_modules,
+    }
+
     if isinstance(data, pd.DataFrame):
         if data.empty:
             raise ValueError("Dataset DataFrame is empty.")
         payload = run_full_analysis(
-            dataset_id=dataset_id,
             dataframe=data,
-            original_filename="<dataframe>",
-            analysis_mode=resolved.mode,
-            target_column=resolved.target,
-            parallel_workers=resolved.workers,
-            skip_ai=True,
-            write_artifacts=resolved.artifacts,
+            **pipeline_kwargs,
         )
     elif isinstance(data, (str, Path)):
         path = _validated_path(data)
+        pipeline_kwargs["original_filename"] = path.name
         payload = run_full_analysis(
-            dataset_id=dataset_id,
             file_path=path,
-            original_filename=path.name,
-            analysis_mode=resolved.mode,
-            target_column=resolved.target,
-            parallel_workers=resolved.workers,
-            skip_ai=True,
-            write_artifacts=resolved.artifacts,
+            **pipeline_kwargs,
         )
     else:
         raise TypeError(
@@ -119,6 +121,7 @@ def plan(
     workers: int | None = None,
     preset: str | None = None,
     config: ConfigInput = None,
+    disabled_modules: list[str] | tuple[str, ...] | None = None,
 ) -> AnalysisPlan:
     """Preview which analyses FrameVitals considers applicable.
 
@@ -133,6 +136,7 @@ def plan(
         target=target,
         workers=workers,
         artifacts=False,
+        disabled_modules=disabled_modules,
     )
     dataframe, source_name = _load_input(data, label="Dataset")
     profile = build_profile(dataframe)
@@ -147,6 +151,25 @@ def plan(
         analysis_mode=resolved.mode,
         target_column=resolved.target,
     )
+    selection["execution_modules"] = {
+        "disabled": sorted(resolved.disabled_modules),
+        "enabled": sorted(
+            name
+            for name in (
+                "deep_statistics",
+                "anomaly_detection",
+                "time_series",
+                "text_profile",
+                "target_intelligence",
+                "modeling",
+                "explainability",
+                "cleaning",
+                "charts",
+                "ai",
+            )
+            if name not in resolved.disabled_modules
+        ),
+    }
 
     public_signals = {
         key: value

From 75975bd0cf80ffc5f85e331f56842382efd95963 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:53:44 +0530
Subject: [PATCH 064/439] feat: expose module controls in public API

---
 src/framevitals/__init__.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 1d49efa..e86c8d1 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -5,7 +5,7 @@
 from typing import Any
 
 from framevitals.cleaning_plan import CleaningPlan
-from framevitals.config import AnalysisConfig
+from framevitals.config import AnalysisConfig, available_modules
 from framevitals.planning import AnalysisPlan
 from framevitals.quality_results import DriftResult, ValidationResult
 from framevitals.result import AnalysisResult, ColumnResult
@@ -28,6 +28,7 @@ def analyze(
     workers: int | None = None,
     preset: str | None = None,
     config: Any = None,
+    disabled_modules: list[str] | tuple[str, ...] | None = None,
 ) -> AnalysisResult:
     """Analyze a tabular dataset from a DataFrame or supported file path."""
     from framevitals.api import analyze as _analyze
@@ -40,6 +41,7 @@ def analyze(
         workers=workers,
         preset=preset,
         config=config,
+        disabled_modules=disabled_modules,
     )
 
 
@@ -51,6 +53,7 @@ def plan(
     workers: int | None = None,
     preset: str | None = None,
     config: Any = None,
+    disabled_modules: list[str] | tuple[str, ...] | None = None,
 ) -> AnalysisPlan:
     """Preview applicable analyses without running heavy analysis stages."""
     from framevitals.api import plan as _plan
@@ -62,6 +65,7 @@ def plan(
         workers=workers,
         preset=preset,
         config=config,
+        disabled_modules=disabled_modules,
     )
 
 
@@ -144,6 +148,7 @@ def validate(data: Any, contract: dict[str, Any]) -> ValidationResult:
     "compare",
     "infer_contract",
     "validate",
+    "available_modules",
     "create_snapshot",
     "load_snapshot",
     "compare_snapshots",

From ae8ddec88fbed6a9e1142c538e6b3a47452a6d63 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:54:35 +0530
Subject: [PATCH 065/439] feat: add CLI module execution controls

---
 src/framevitals/cli.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/cli.py b/src/framevitals/cli.py
index 4bcb803..af1e241 100644
--- a/src/framevitals/cli.py
+++ b/src/framevitals/cli.py
@@ -3,7 +3,7 @@
 from pathlib import Path
 
 from framevitals import __version__
-from framevitals.config import available_presets
+from framevitals.config import available_modules, available_presets
 
 
 def _add_output_argument(parser: argparse.ArgumentParser) -> None:
@@ -45,6 +45,17 @@ def _add_runtime_arguments(parser: argparse.ArgumentParser) -> None:
         default=None,
         help="Parallel worker count. Overrides config values.",
     )
+    parser.add_argument(
+        "--disable-module",
+        dest="disabled_modules",
+        action="append",
+        choices=list(available_modules()),
+        default=None,
+        help=(
+            "Disable an optional execution module. Repeat the flag to disable "
+            "multiple modules. Explicit flags override the config/preset list."
+        ),
+    )
 
 
 def _load_contract(path: Path) -> dict:
@@ -337,6 +348,7 @@ def main() -> int:
             workers=args.workers,
             preset=args.preset,
             config=args.config,
+            disabled_modules=args.disabled_modules,
         )
 
         if args.output is not None:
@@ -356,6 +368,9 @@ def main() -> int:
                     f"workers={resolved.get('workers')} "
                     f"artifacts={resolved.get('artifacts')}"
                 )
+                disabled = resolved.get("disabled_modules") or ()
+                if disabled:
+                    print(f"Disabled      {', '.join(disabled)}")
             if args.output is not None:
                 print(f"Full JSON     {args.output}")
             if args.html_report is not None:
@@ -372,6 +387,7 @@ def main() -> int:
             workers=args.workers,
             preset=args.preset,
             config=args.config,
+            disabled_modules=args.disabled_modules,
         )
         if args.output is not None:
             _write_json(dict(result), args.output)
@@ -379,6 +395,9 @@ def main() -> int:
             print(json.dumps(dict(result), indent=2, default=str))
         else:
             print(result.explain_text())
+            disabled = result.get("config", {}).get("disabled_modules") or ()
+            if disabled:
+                print(f"Disabled modules: {', '.join(disabled)}")
             if args.output is not None:
                 print(f"Plan JSON     {args.output}")
         return 0

From 5dcd7d3ab452f110dd8ea11c3cc82ce93a3aee3a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:55:17 +0530
Subject: [PATCH 066/439] fix: preserve module precedence across presets and
 TOML overrides

---
 src/framevitals/config.py | 53 ++++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/src/framevitals/config.py b/src/framevitals/config.py
index fc8201c..56a0f75 100644
--- a/src/framevitals/config.py
+++ b/src/framevitals/config.py
@@ -118,8 +118,10 @@ def _coerce_disabled_modules(value: Any) -> tuple[str, ...]:
     return tuple(str(item) for item in value)
 
 
-def _extract_values(mapping: Mapping[str, Any]) -> tuple[str | None, dict[str, Any]]:
-    """Extract supported values from nested or flat configuration mappings."""
+def _extract_values(
+    mapping: Mapping[str, Any],
+) -> tuple[str | None, dict[str, Any], dict[str, bool]]:
+    """Extract scalar values and module overrides from config mappings."""
     analysis = mapping.get("analysis", {})
     resources = mapping.get("resources", {})
     modules = mapping.get("modules", {})
@@ -145,26 +147,24 @@ def _extract_values(mapping: Mapping[str, Any]) -> tuple[str | None, dict[str, A
     elif "workers" in mapping:
         values["workers"] = mapping["workers"]
 
-    raw_disabled = analysis.get(
-        "disabled_modules",
-        mapping.get("disabled_modules", ()),
-    )
-    disabled = list(_coerce_disabled_modules(raw_disabled))
+    if "disabled_modules" in analysis:
+        values["disabled_modules"] = _coerce_disabled_modules(
+            analysis["disabled_modules"]
+        )
+    elif "disabled_modules" in mapping:
+        values["disabled_modules"] = _coerce_disabled_modules(
+            mapping["disabled_modules"]
+        )
 
+    module_overrides: dict[str, bool] = {}
     for name, enabled in modules.items():
         if name not in VALID_MODULES:
             raise ValueError(f"Unknown FrameVitals module in [modules]: {name}")
         if not isinstance(enabled, bool):
             raise ValueError(f"[modules].{name} must be true or false.")
-        if enabled:
-            disabled = [item for item in disabled if item != name]
-        elif name not in disabled:
-            disabled.append(name)
-
-    if disabled:
-        values["disabled_modules"] = tuple(disabled)
+        module_overrides[name] = enabled
 
-    return str(preset) if preset is not None else None, values
+    return str(preset) if preset is not None else None, values, module_overrides
 
 
 def _preset_values(name: str | None) -> dict[str, Any]:
@@ -178,6 +178,19 @@ def _preset_values(name: str | None) -> dict[str, Any]:
     return dict(PRESETS[name])
 
 
+def _apply_module_overrides(
+    values: dict[str, Any],
+    overrides: Mapping[str, bool],
+) -> None:
+    disabled = list(_coerce_disabled_modules(values.get("disabled_modules", ())))
+    for name, enabled in overrides.items():
+        if enabled:
+            disabled = [item for item in disabled if item != name]
+        elif name not in disabled:
+            disabled.append(name)
+    values["disabled_modules"] = tuple(disabled)
+
+
 def resolve_config(
     config: ConfigInput = None,
     *,
@@ -194,7 +207,7 @@ def resolve_config(
 
     1. FrameVitals defaults
     2. explicit ``preset=`` argument
-    3. configuration file/mapping/object
+    3. configuration file/mapping/object (including ``[modules]`` booleans)
     4. explicit function/CLI arguments
     """
     values: dict[str, Any] = AnalysisConfig().to_dict()
@@ -203,15 +216,19 @@ def resolve_config(
     if isinstance(config, AnalysisConfig):
         values.update(config.to_dict())
     elif isinstance(config, (str, Path)):
-        config_preset, config_values = _extract_values(_read_toml(config))
+        config_preset, config_values, module_overrides = _extract_values(
+            _read_toml(config)
+        )
         if config_preset is not None:
             values.update(_preset_values(config_preset))
         values.update(config_values)
+        _apply_module_overrides(values, module_overrides)
     elif isinstance(config, Mapping):
-        config_preset, config_values = _extract_values(config)
+        config_preset, config_values, module_overrides = _extract_values(config)
         if config_preset is not None:
             values.update(_preset_values(config_preset))
         values.update(config_values)
+        _apply_module_overrides(values, module_overrides)
     elif config is not None:
         raise TypeError(
             "config must be an AnalysisConfig, mapping, TOML path, or None."

From 6b30b5d9c2db17b69ea0d0bea62109441307efa7 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:55:45 +0530
Subject: [PATCH 067/439] test: cover module config and preset precedence

---
 tests/test_config.py | 51 ++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 49 insertions(+), 2 deletions(-)

diff --git a/tests/test_config.py b/tests/test_config.py
index 2fd1a2d..1a780bd 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -2,7 +2,7 @@
 import pytest
 
 import framevitals
-from framevitals.config import AnalysisConfig, resolve_config
+from framevitals.config import AnalysisConfig, available_modules, resolve_config
 
 
 def test_default_and_preset_resolution():
@@ -12,6 +12,17 @@ def test_default_and_preset_resolution():
     assert quick.mode == "quick"
     assert quick.workers == 2
     assert quick.artifacts is False
+    assert quick.disabled_modules == ()
+
+    ci = resolve_config(preset="ci")
+    assert ci.mode == "standard"
+    assert ci.workers == 2
+    assert set(ci.disabled_modules) == {
+        "modeling",
+        "explainability",
+        "charts",
+        "ai",
+    }
 
 
 def test_toml_config_resolution_and_explicit_precedence(tmp_path):
@@ -21,9 +32,13 @@ def test_toml_config_resolution_and_explicit_precedence(tmp_path):
         "preset = \"deep\"\n"
         "target = \"churn\"\n"
         "artifacts = true\n"
+        "disabled_modules = [\"text_profile\"]\n"
         "\n"
         "[resources]\n"
-        "workers = 6\n",
+        "workers = 6\n"
+        "\n"
+        "[modules]\n"
+        "anomaly_detection = false\n",
         encoding="utf-8",
     )
 
@@ -32,6 +47,7 @@ def test_toml_config_resolution_and_explicit_precedence(tmp_path):
     assert resolved.target == "churn"
     assert resolved.artifacts is True
     assert resolved.workers == 6
+    assert resolved.disabled_modules == ("text_profile", "anomaly_detection")
 
     overridden = resolve_config(
         config,
@@ -39,27 +55,56 @@ def test_toml_config_resolution_and_explicit_precedence(tmp_path):
         target="label",
         artifacts=False,
         workers=3,
+        disabled_modules=["charts"],
     )
     assert overridden == AnalysisConfig(
         mode="quick",
         target="label",
         artifacts=False,
         workers=3,
+        disabled_modules=("charts",),
+    )
+
+
+def test_module_booleans_can_selectively_override_preset_defaults():
+    resolved = resolve_config(
+        preset="ci",
+        config={
+            "modules": {
+                "modeling": True,
+                "time_series": False,
+            },
+        },
     )
 
+    assert "modeling" not in resolved.disabled_modules
+    assert "time_series" in resolved.disabled_modules
+    assert "explainability" in resolved.disabled_modules
+    assert "charts" in resolved.disabled_modules
+    assert "ai" in resolved.disabled_modules
+
 
 def test_config_mapping_and_validation():
     resolved = resolve_config({
         "analysis": {"mode": "research", "artifacts": False},
         "resources": {"workers": 8},
+        "modules": {"deep_statistics": False},
     })
     assert resolved.mode == "research"
     assert resolved.workers == 8
+    assert resolved.disabled_modules == ("deep_statistics",)
+
+    assert "modeling" in available_modules()
+    assert "cleaning" in available_modules()
 
     with pytest.raises(ValueError, match="Unknown FrameVitals preset"):
         resolve_config(preset="everything")
     with pytest.raises(ValueError, match="workers"):
         resolve_config(workers=0)
+    with pytest.raises(ValueError, match="Unknown FrameVitals module"):
+        resolve_config(disabled_modules=["magic_model"])
+    with pytest.raises(ValueError, match="Unknown FrameVitals module in"):
+        resolve_config({"modules": {"magic_model": False}})
 
 
 def test_public_analyze_honors_config_and_records_resolution(tmp_path, monkeypatch):
@@ -84,5 +129,7 @@ def test_public_analyze_honors_config_and_records_resolution(tmp_path, monkeypat
         "target": None,
         "artifacts": False,
         "workers": 1,
+        "disabled_modules": (),
     }
+    assert result["execution"]["disabled_modules"] == []
     assert not (tmp_path / "cleaned").exists()

From f174a0d4a34958b4fb9b3d3ec95e61063001fccb Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:56:11 +0530
Subject: [PATCH 068/439] test: verify configurable execution modules

---
 tests/test_execution_controls.py | 131 +++++++++++++++++++++++++++++++
 1 file changed, 131 insertions(+)
 create mode 100644 tests/test_execution_controls.py

diff --git a/tests/test_execution_controls.py b/tests/test_execution_controls.py
new file mode 100644
index 0000000..4984ea5
--- /dev/null
+++ b/tests/test_execution_controls.py
@@ -0,0 +1,131 @@
+import pandas as pd
+
+import framevitals
+from framevitals.cli import build_parser, main
+
+
+def _frame(rows: int = 36) -> pd.DataFrame:
+    return pd.DataFrame({
+        "value": list(range(rows)),
+        "value_2": [index * 2 for index in range(rows)],
+        "segment": ["A", "B", "C"] * (rows // 3),
+        "target": [0, 1] * (rows // 2),
+    })
+
+
+def test_analyze_can_disable_expensive_modules_without_losing_result_keys(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    result = framevitals.analyze(
+        _frame(),
+        target="target",
+        mode="standard",
+        artifacts=True,
+        disabled_modules=[
+            "deep_statistics",
+            "anomaly_detection",
+            "time_series",
+            "text_profile",
+            "target_intelligence",
+            "modeling",
+            "explainability",
+            "cleaning",
+            "charts",
+            "ai",
+        ],
+    )
+
+    assert result["execution"]["disabled_modules"] == sorted(
+        framevitals.available_modules()
+    )
+    assert all(
+        status == "disabled"
+        for status in result["execution"]["module_status"].values()
+    )
+
+    assert result["deep_statistics_v2"]["skipped"] is True
+    assert result["anomalies_v2"]["skipped"] is True
+    assert result["time_series"]["skipped"] is True
+    assert result["text_profile"]["skipped"] is True
+    assert result["target_intelligence"]["skipped"] is True
+    assert result["model_leaderboard"]["skipped"] is True
+    assert result["explainability"]["skipped"] is True
+    assert result["cleaning"]["skipped"] is True
+    assert result["charts"] == []
+    assert result["ai_report"]["source"] == "disabled"
+
+    assert not (tmp_path / "cleaned").exists()
+    assert not (tmp_path / "static" / "charts").exists()
+
+
+def test_ci_preset_skips_modeling_but_keeps_target_intelligence():
+    result = framevitals.analyze(
+        _frame(),
+        target="target",
+        preset="ci",
+    )
+
+    execution = result["execution"]["module_status"]
+    assert execution["modeling"] == "disabled"
+    assert execution["explainability"] == "disabled"
+    assert execution["charts"] == "disabled"
+    assert execution["ai"] == "disabled"
+    assert execution["target_intelligence"] == "ran"
+
+    assert result["model_leaderboard"]["skipped"] is True
+    assert result["target_intelligence"]["available"] is True
+
+
+def test_plan_surfaces_execution_module_selection():
+    plan = framevitals.plan(
+        _frame(),
+        target="target",
+        mode="deep",
+        disabled_modules=["modeling", "charts"],
+    )
+
+    modules = plan["selection"]["execution_modules"]
+    assert modules["disabled"] == ["charts", "modeling"]
+    assert "anomaly_detection" in modules["enabled"]
+    assert plan["config"]["disabled_modules"] == ("modeling", "charts")
+
+
+def test_cli_accepts_repeatable_disable_module_flags():
+    parser = build_parser()
+    args = parser.parse_args([
+        "analyze",
+        "data.csv",
+        "--disable-module",
+        "modeling",
+        "--disable-module",
+        "charts",
+    ])
+
+    assert args.disabled_modules == ["modeling", "charts"]
+
+
+def test_cli_module_flags_reach_pipeline(tmp_path, monkeypatch, capsys):
+    dataset = tmp_path / "data.csv"
+    _frame().to_csv(dataset, index=False)
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "analyze",
+            str(dataset),
+            "--mode",
+            "standard",
+            "--disable-module",
+            "anomaly_detection",
+            "--disable-module",
+            "modeling",
+            "--format",
+            "json",
+        ],
+    )
+
+    assert main() == 0
+    payload = __import__("json").loads(capsys.readouterr().out)
+    assert payload["execution"]["module_status"]["anomaly_detection"] == "disabled"
+    assert payload["execution"]["module_status"]["modeling"] == "disabled"
+    assert payload["anomalies_v2"]["skipped"] is True

From 5f95fd4eaa736dabf41db0ab4b0d550d8f48a9c2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:57:34 +0530
Subject: [PATCH 069/439] fix: keep top-level package import dependency-light

---
 src/framevitals/__init__.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index e86c8d1..3d6f3dc 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -2,9 +2,8 @@
 
 from __future__ import annotations
 
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
-from framevitals.cleaning_plan import CleaningPlan
 from framevitals.config import AnalysisConfig, available_modules
 from framevitals.planning import AnalysisPlan
 from framevitals.quality_results import DriftResult, ValidationResult
@@ -16,9 +15,26 @@
     load_snapshot,
 )
 
+if TYPE_CHECKING:
+    from framevitals.cleaning_plan import CleaningPlan
+
 __version__ = "0.1.0"
 
 
+def __getattr__(name: str):
+    """Lazily expose optional/heavier public types.
+
+    ``CleaningPlan`` depends on pandas/NumPy. Delaying that import preserves the
+    package's lightweight top-level import boundary and built-wheel smoke test
+    while keeping ``framevitals.CleaningPlan`` available to normal users.
+    """
+    if name == "CleaningPlan":
+        from framevitals.cleaning_plan import CleaningPlan
+
+        return CleaningPlan
+    raise AttributeError(name)
+
+
 def analyze(
     data: Any,
     *,

From 0a82e3294e421ae02f2725347255109fffce7a85 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 09:58:23 +0530
Subject: [PATCH 070/439] test: distinguish disabled from not-applicable
 modules

---
 tests/test_execution_controls.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/test_execution_controls.py b/tests/test_execution_controls.py
index 4984ea5..5887d5f 100644
--- a/tests/test_execution_controls.py
+++ b/tests/test_execution_controls.py
@@ -127,5 +127,10 @@ def test_cli_module_flags_reach_pipeline(tmp_path, monkeypatch, capsys):
     assert main() == 0
     payload = __import__("json").loads(capsys.readouterr().out)
     assert payload["execution"]["module_status"]["anomaly_detection"] == "disabled"
-    assert payload["execution"]["module_status"]["modeling"] == "disabled"
     assert payload["anomalies_v2"]["skipped"] is True
+
+    # Modeling requires a target in the first place. The configuration still
+    # records it as disabled, but execution correctly explains that this run
+    # skipped it because it was not applicable.
+    assert "modeling" in payload["execution"]["disabled_modules"]
+    assert payload["execution"]["module_status"]["modeling"] == "not_applicable"

From bb64a30368963d65e6273c5ac939dcf2ba02bf9f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:01:15 +0530
Subject: [PATCH 071/439] feat: add bounded deterministic quality diagnostics

---
 src/framevitals/quality_diagnostics.py | 537 +++++++++++++++++++++++++
 1 file changed, 537 insertions(+)
 create mode 100644 src/framevitals/quality_diagnostics.py

diff --git a/src/framevitals/quality_diagnostics.py b/src/framevitals/quality_diagnostics.py
new file mode 100644
index 0000000..29ded4a
--- /dev/null
+++ b/src/framevitals/quality_diagnostics.py
@@ -0,0 +1,537 @@
+"""Deterministic data-quality diagnostics for tabular datasets.
+
+The diagnostics in this module are intentionally explainable and resource
+bounded. Existing profile/role metadata is reused where possible. Potentially
+expensive value-level checks operate on a deterministic bounded sample, while
+exact full-column comparisons are reserved for duplicate-column candidates
+identified by a sample fingerprint first.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from itertools import combinations
+from typing import Any, Mapping
+
+import numpy as np
+import pandas as pd
+
+
+DEFAULT_MAX_SAMPLE_ROWS = 5_000
+DEFAULT_MAX_COLUMNS = 100
+DEFAULT_MAX_MISSINGNESS_COLUMNS = 25
+
+
+def _bounded_series(series: pd.Series, max_rows: int) -> tuple[pd.Series, bool]:
+    if len(series) <= max_rows:
+        return series, False
+    positions = np.linspace(0, len(series) - 1, num=max_rows, dtype=int)
+    positions = np.unique(positions)
+    return series.iloc[positions], True
+
+
+def _bounded_frame(frame: pd.DataFrame, max_rows: int) -> tuple[pd.DataFrame, bool]:
+    if len(frame) <= max_rows:
+        return frame, False
+    positions = np.linspace(0, len(frame) - 1, num=max_rows, dtype=int)
+    positions = np.unique(positions)
+    return frame.iloc[positions], True
+
+
+def _text_like(series: pd.Series) -> bool:
+    return bool(
+        pd.api.types.is_object_dtype(series)
+        or pd.api.types.is_string_dtype(series.dtype)
+        or isinstance(series.dtype, pd.CategoricalDtype)
+    )
+
+
+def _column_info(column_roles: Mapping[str, Any], column: str) -> dict[str, Any]:
+    value = column_roles.get(column, {})
+    return dict(value) if isinstance(value, Mapping) else {}
+
+
+def _profile_mapping(profile: Mapping[str, Any], key: str) -> dict[str, Any]:
+    value = profile.get(key, {})
+    return dict(value) if isinstance(value, Mapping) else {}
+
+
+def _primary_key_candidates(
+    df: pd.DataFrame,
+    column_roles: Mapping[str, Any],
+) -> list[dict[str, Any]]:
+    rows = len(df)
+    if rows < 2:
+        return []
+
+    candidates: list[dict[str, Any]] = []
+    for column in df.columns:
+        info = _column_info(column_roles, column)
+        unique_count = int(info.get("unique_count") or 0)
+        non_missing = int(info.get("non_missing_count") or 0)
+        if unique_count != rows or non_missing != rows:
+            continue
+
+        roles = set(info.get("roles", []))
+        confidence = "high" if "id_like" in roles else "medium"
+        candidates.append({
+            "column": column,
+            "confidence": confidence,
+            "reason": (
+                "Column is complete, unique for every row, and identifier-like."
+                if confidence == "high"
+                else "Column is complete and unique for every row."
+            ),
+        })
+
+    candidates.sort(key=lambda item: (item["confidence"] != "high", item["column"]))
+    return candidates
+
+
+def _identifier_duplicates(
+    df: pd.DataFrame,
+    column_roles: Mapping[str, Any],
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+    for column in df.columns:
+        info = _column_info(column_roles, column)
+        if "id_like" not in set(info.get("roles", [])):
+            continue
+        clean = df[column].dropna()
+        if clean.empty:
+            continue
+        duplicate_mask = clean.duplicated(keep=False)
+        duplicate_rows = int(duplicate_mask.sum())
+        if not duplicate_rows:
+            continue
+        duplicated_values = int(clean.loc[duplicate_mask].nunique(dropna=True))
+        results.append({
+            "column": column,
+            "duplicate_rows": duplicate_rows,
+            "duplicated_identifier_values": duplicated_values,
+            "severity": "high",
+        })
+    return results
+
+
+def _quasi_constants(
+    df: pd.DataFrame,
+    column_roles: Mapping[str, Any],
+    *,
+    max_rows: int,
+    threshold: float = 0.95,
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+    for column in df.columns:
+        info = _column_info(column_roles, column)
+        if int(info.get("unique_count") or 0) <= 1:
+            continue
+        sample, sampled = _bounded_series(df[column], max_rows)
+        clean = sample.dropna()
+        if len(clean) < 10:
+            continue
+        counts = clean.value_counts(dropna=True)
+        if counts.empty:
+            continue
+        ratio = float(counts.iloc[0] / len(clean))
+        if ratio < threshold:
+            continue
+        results.append({
+            "column": column,
+            "top_value": str(counts.index[0]),
+            "top_value_ratio": round(ratio, 4),
+            "sampled": sampled,
+            "sample_rows": int(len(sample)),
+            "severity": "medium" if ratio >= 0.99 else "low",
+        })
+    return results
+
+
+def _series_fingerprint(series: pd.Series) -> str:
+    hashed = pd.util.hash_pandas_object(series, index=False).to_numpy(dtype="uint64")
+    digest = hashlib.blake2b(digest_size=16)
+    digest.update(str(series.dtype).encode("utf-8"))
+    digest.update(hashed.tobytes())
+    return digest.hexdigest()
+
+
+def _duplicate_columns(
+    df: pd.DataFrame,
+    *,
+    max_rows: int,
+    max_columns: int,
+) -> list[dict[str, Any]]:
+    columns = list(df.columns[:max_columns])
+    sample, sampled = _bounded_frame(df[columns], max_rows)
+    buckets: dict[str, list[str]] = {}
+
+    for column in columns:
+        fingerprint = _series_fingerprint(sample[column])
+        buckets.setdefault(fingerprint, []).append(column)
+
+    results: list[dict[str, Any]] = []
+    for candidates in buckets.values():
+        if len(candidates) < 2:
+            continue
+        anchor = candidates[0]
+        duplicates = [
+            candidate
+            for candidate in candidates[1:]
+            if df[anchor].equals(df[candidate])
+        ]
+        if duplicates:
+            results.append({
+                "canonical_column": anchor,
+                "duplicate_columns": duplicates,
+                "sampled_for_fingerprint": sampled,
+                "confirmed_with_full_equality": True,
+                "severity": "medium",
+            })
+    return results
+
+
+def _normalise_numeric_text(values: pd.Series) -> pd.Series:
+    cleaned = values.str.replace(",", "", regex=False).str.strip()
+    cleaned = cleaned.str.replace(r"^[\$€£₹]\s*", "", regex=True)
+    cleaned = cleaned.str.replace(r"%$", "", regex=True)
+    return cleaned
+
+
+def _coercion_candidates(
+    df: pd.DataFrame,
+    column_roles: Mapping[str, Any],
+    *,
+    max_rows: int,
+    threshold: float = 0.95,
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+
+    for column in df.columns:
+        series = df[column]
+        if not _text_like(series):
+            continue
+
+        info = _column_info(column_roles, column)
+        roles = set(info.get("roles", []))
+        sample, sampled = _bounded_series(series, max_rows)
+        clean = sample.dropna().astype("string").str.strip()
+        clean = clean[clean.ne("")]
+        if len(clean) < 10:
+            continue
+
+        numeric_values = _normalise_numeric_text(clean)
+        numeric_ratio = float(pd.to_numeric(numeric_values, errors="coerce").notna().mean())
+
+        if numeric_ratio >= threshold and "id_like" not in roles:
+            semantic = info.get("semantic_type")
+            transform = "parse numeric strings"
+            if semantic == "currency":
+                transform = "remove currency symbols/separators, then parse numeric"
+            elif semantic == "percentage":
+                transform = "remove percent suffix, then parse numeric"
+            results.append({
+                "column": column,
+                "suggested_type": "numeric",
+                "parse_ratio": round(numeric_ratio, 4),
+                "transform": transform,
+                "sampled": sampled,
+                "sample_rows": int(len(sample)),
+                "severity": "low",
+            })
+            continue
+
+        if numeric_ratio >= 0.50:
+            # Avoid treating mostly-numeric identifiers as dates.
+            continue
+
+        parsed_dates = pd.to_datetime(clean, errors="coerce", format="mixed")
+        date_ratio = float(parsed_dates.notna().mean())
+        if date_ratio >= threshold:
+            results.append({
+                "column": column,
+                "suggested_type": "datetime",
+                "parse_ratio": round(date_ratio, 4),
+                "transform": "parse datetime strings",
+                "sampled": sampled,
+                "sample_rows": int(len(sample)),
+                "severity": "low",
+            })
+
+    return results
+
+
+def _category_normalisation_issues(
+    df: pd.DataFrame,
+    column_roles: Mapping[str, Any],
+    *,
+    max_rows: int,
+    max_unique: int = 200,
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+
+    for column in df.columns:
+        series = df[column]
+        if not _text_like(series):
+            continue
+        info = _column_info(column_roles, column)
+        if "long_text" in set(info.get("roles", [])):
+            continue
+
+        sample, sampled = _bounded_series(series, max_rows)
+        clean = sample.dropna().astype("string")
+        unique_values = clean.unique().tolist()
+        if len(unique_values) < 2 or len(unique_values) > max_unique:
+            continue
+
+        groups: dict[str, list[str]] = {}
+        for raw in unique_values:
+            raw_text = str(raw)
+            canonical = raw_text.strip().casefold()
+            groups.setdefault(canonical, []).append(raw_text)
+
+        variants = [
+            {"canonical": canonical, "variants": sorted(set(raw_values))}
+            for canonical, raw_values in groups.items()
+            if len(set(raw_values)) > 1
+        ]
+        if not variants:
+            continue
+
+        variants.sort(key=lambda item: (-len(item["variants"]), item["canonical"]))
+        results.append({
+            "column": column,
+            "variant_group_count": len(variants),
+            "groups": variants[:10],
+            "sampled": sampled,
+            "sample_rows": int(len(sample)),
+            "severity": "medium",
+        })
+
+    return results
+
+
+def _blank_string_issues(
+    df: pd.DataFrame,
+    *,
+    max_rows: int,
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+    for column in df.columns:
+        series = df[column]
+        if not _text_like(series):
+            continue
+        sample, sampled = _bounded_series(series, max_rows)
+        non_null = sample.dropna().astype("string")
+        if non_null.empty:
+            continue
+        blank_count = int(non_null.str.strip().eq("").sum())
+        if not blank_count:
+            continue
+        results.append({
+            "column": column,
+            "blank_count_in_sample": blank_count,
+            "blank_ratio_in_sample": round(float(blank_count / len(non_null)), 4),
+            "sampled": sampled,
+            "sample_rows": int(len(sample)),
+            "severity": "medium",
+        })
+    return results
+
+
+def _infinity_issues(
+    df: pd.DataFrame,
+    *,
+    max_rows: int,
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+    numeric_columns = df.select_dtypes(include=[np.number]).columns
+    for column in numeric_columns:
+        sample, sampled = _bounded_series(df[column], max_rows)
+        values = pd.to_numeric(sample, errors="coerce").to_numpy(dtype="float64")
+        count = int(np.isinf(values).sum())
+        if not count:
+            continue
+        results.append({
+            "column": column,
+            "infinite_count_in_sample": count,
+            "sampled": sampled,
+            "sample_rows": int(len(sample)),
+            "severity": "high",
+        })
+    return results
+
+
+def _mixed_object_types(
+    df: pd.DataFrame,
+    *,
+    max_rows: int,
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+    for column in df.columns:
+        series = df[column]
+        if not pd.api.types.is_object_dtype(series):
+            continue
+        sample, sampled = _bounded_series(series, max_rows)
+        clean = sample.dropna()
+        if len(clean) < 2:
+            continue
+        counts = clean.map(lambda value: type(value).__name__).value_counts()
+        if len(counts) <= 1:
+            continue
+        results.append({
+            "column": column,
+            "python_types": {str(key): int(value) for key, value in counts.items()},
+            "sampled": sampled,
+            "sample_rows": int(len(sample)),
+            "severity": "medium",
+        })
+    return results
+
+
+def _missingness_relationships(
+    df: pd.DataFrame,
+    profile: Mapping[str, Any],
+    *,
+    max_rows: int,
+    max_columns: int,
+    jaccard_threshold: float = 0.70,
+) -> list[dict[str, Any]]:
+    missing_counts = _profile_mapping(profile, "missing_counts")
+    candidates = [
+        column
+        for column in df.columns
+        if int(missing_counts.get(column) or 0) > 0
+    ][:max_columns]
+
+    if len(candidates) < 2:
+        return []
+
+    sample, sampled = _bounded_frame(df[candidates], max_rows)
+    masks = {column: sample[column].isna().to_numpy() for column in candidates}
+    relationships: list[dict[str, Any]] = []
+
+    for left, right in combinations(candidates, 2):
+        left_mask = masks[left]
+        right_mask = masks[right]
+        union = int(np.logical_or(left_mask, right_mask).sum())
+        if not union:
+            continue
+        intersection = int(np.logical_and(left_mask, right_mask).sum())
+        if intersection < 3:
+            continue
+        jaccard = intersection / union
+        if jaccard < jaccard_threshold:
+            continue
+        relationships.append({
+            "columns": [left, right],
+            "co_missing_rows_in_sample": intersection,
+            "jaccard": round(float(jaccard), 4),
+            "sampled": sampled,
+            "sample_rows": int(len(sample)),
+            "severity": "medium" if jaccard >= 0.90 else "low",
+        })
+
+    relationships.sort(key=lambda item: (-item["jaccard"], item["columns"]))
+    return relationships[:30]
+
+
+def run_quality_diagnostics(
+    df: pd.DataFrame,
+    *,
+    profile: Mapping[str, Any] | None = None,
+    column_roles: Mapping[str, Any] | None = None,
+    max_sample_rows: int = DEFAULT_MAX_SAMPLE_ROWS,
+    max_columns: int = DEFAULT_MAX_COLUMNS,
+    max_missingness_columns: int = DEFAULT_MAX_MISSINGNESS_COLUMNS,
+) -> dict[str, Any]:
+    """Return a bounded, deterministic set of practical data-quality checks."""
+    if max_sample_rows < 10:
+        raise ValueError("max_sample_rows must be at least 10.")
+    if max_columns < 1:
+        raise ValueError("max_columns must be at least 1.")
+    if max_missingness_columns < 2:
+        raise ValueError("max_missingness_columns must be at least 2.")
+
+    if profile is None:
+        from framevitals.profiler import build_profile
+
+        profile = build_profile(df)
+    if column_roles is None:
+        from framevitals.column_roles import infer_column_roles
+
+        column_roles = infer_column_roles(df)
+
+    selected_columns = list(df.columns[:max_columns])
+    work = df[selected_columns]
+    selected_roles = {
+        column: _column_info(column_roles, column)
+        for column in selected_columns
+    }
+
+    primary_keys = _primary_key_candidates(work, selected_roles)
+    identifier_duplicates = _identifier_duplicates(work, selected_roles)
+    quasi_constants = _quasi_constants(
+        work,
+        selected_roles,
+        max_rows=max_sample_rows,
+    )
+    duplicate_columns = _duplicate_columns(
+        work,
+        max_rows=max_sample_rows,
+        max_columns=max_columns,
+    )
+    coercions = _coercion_candidates(
+        work,
+        selected_roles,
+        max_rows=max_sample_rows,
+    )
+    category_normalisation = _category_normalisation_issues(
+        work,
+        selected_roles,
+        max_rows=max_sample_rows,
+    )
+    blank_strings = _blank_string_issues(work, max_rows=max_sample_rows)
+    infinities = _infinity_issues(work, max_rows=max_sample_rows)
+    mixed_types = _mixed_object_types(work, max_rows=max_sample_rows)
+    missingness_relationships = _missingness_relationships(
+        work,
+        profile,
+        max_rows=max_sample_rows,
+        max_columns=max_missingness_columns,
+    )
+
+    duplicate_rows = int(profile.get("duplicate_rows", 0) or 0)
+    checks = {
+        "primary_key_candidates": primary_keys,
+        "identifier_duplicates": identifier_duplicates,
+        "quasi_constant_columns": quasi_constants,
+        "duplicate_columns": duplicate_columns,
+        "coercion_candidates": coercions,
+        "category_normalisation": category_normalisation,
+        "blank_strings": blank_strings,
+        "infinite_values": infinities,
+        "mixed_object_types": mixed_types,
+        "missingness_relationships": missingness_relationships,
+    }
+
+    issue_count = sum(
+        len(value)
+        for key, value in checks.items()
+        if key != "primary_key_candidates"
+    )
+
+    return {
+        "available": True,
+        "rows": int(len(df)),
+        "columns": int(len(df.columns)),
+        "columns_checked": len(selected_columns),
+        "truncated_columns": len(df.columns) > len(selected_columns),
+        "max_sample_rows": int(max_sample_rows),
+        "duplicate_rows": duplicate_rows,
+        "summary": {
+            "issue_groups": sum(bool(value) for key, value in checks.items() if key != "primary_key_candidates"),
+            "issue_count": issue_count + (1 if duplicate_rows else 0),
+            "primary_key_candidate_count": len(primary_keys),
+        },
+        **checks,
+    }

From 49f602865eae4c68c9d771065fa07dc1103d2b9d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:01:53 +0530
Subject: [PATCH 072/439] feat: make quality diagnostics configurable

---
 src/framevitals/config.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/framevitals/config.py b/src/framevitals/config.py
index 56a0f75..1f4c10a 100644
--- a/src/framevitals/config.py
+++ b/src/framevitals/config.py
@@ -16,6 +16,7 @@
 
 VALID_MODES = {"quick", "standard", "deep", "research"}
 VALID_MODULES = {
+    "quality_diagnostics",
     "deep_statistics",
     "anomaly_detection",
     "time_series",

From ac107e0cbc3ef0cf28f6e963409587621dfcacb7 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:03:12 +0530
Subject: [PATCH 073/439] feat: integrate configurable quality diagnostics into
 analysis

---
 src/framevitals/pipeline.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/pipeline.py b/src/framevitals/pipeline.py
index 4215f34..9bb1d39 100644
--- a/src/framevitals/pipeline.py
+++ b/src/framevitals/pipeline.py
@@ -32,6 +32,7 @@
 from framevitals.ml_readiness import calculate_ml_readiness
 from framevitals.model_leaderboard import run_model_leaderboard
 from framevitals.profiler import build_profile
+from framevitals.quality_diagnostics import run_quality_diagnostics
 from framevitals.signal_engine import build_signals
 from framevitals.target_intelligence import run_target_intelligence
 from framevitals.text_profile import profile_text_columns
@@ -147,7 +148,7 @@ def module_enabled(name: str) -> bool:
     }
     timings_ms["analysis_selection"] = (time.perf_counter() - t0) * 1000
 
-    # Phase 2: quality scoring remains core and always available.
+    # Phase 2: core quality/readiness plus bounded practical diagnostics.
     t0 = time.perf_counter()
     health = calculate_health_score(df, profile)
     timings_ms["health"] = (time.perf_counter() - t0) * 1000
@@ -160,6 +161,24 @@ def module_enabled(name: str) -> bool:
     advanced = calculate_advanced_indicators(df)
     timings_ms["advanced"] = (time.perf_counter() - t0) * 1000
 
+    if module_enabled("quality_diagnostics"):
+        max_sample_rows = 1_000 if analysis_mode == "quick" else 5_000
+        _, quality_diagnostics, quality_elapsed = _safe_call(
+            "quality_diagnostics",
+            lambda: run_quality_diagnostics(
+                df,
+                profile=profile,
+                column_roles=column_roles,
+                max_sample_rows=max_sample_rows,
+            ),
+        )
+        timings_ms["quality_diagnostics"] = quality_elapsed
+        module_status["quality_diagnostics"] = _result_status(quality_diagnostics)
+    else:
+        quality_diagnostics = _skipped_module("quality_diagnostics")
+        timings_ms["quality_diagnostics"] = 0.0
+        module_status["quality_diagnostics"] = "disabled"
+
     # Phase 3: independent heavier analyses.
     deep_statistics_v2 = None
     anomalies_v2 = None
@@ -424,6 +443,7 @@ def module_enabled(name: str) -> bool:
         "signals": signals,
         "ml_readiness": ml_readiness,
         "advanced": advanced,
+        "quality_diagnostics": quality_diagnostics,
         "deep_statistics_v2": deep_statistics_v2,
         "anomalies_v2": anomalies_v2,
         "target_intelligence": target_intelligence,

From 09f0504fc1390cc32231a391e5193af1e684a272 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:03:46 +0530
Subject: [PATCH 074/439] feat: normalize quality diagnostics into findings

---
 src/framevitals/findings.py | 178 ++++++++++++++++++++++++++++++++++++
 1 file changed, 178 insertions(+)

diff --git a/src/framevitals/findings.py b/src/framevitals/findings.py
index ed1418f..f484841 100644
--- a/src/framevitals/findings.py
+++ b/src/framevitals/findings.py
@@ -151,6 +151,184 @@ def findings_from_target_intelligence(
     return _sort_findings(findings)
 
 
+def _quality_finding(
+    *,
+    code: str,
+    title: str,
+    severity: Any,
+    evidence: str,
+    recommendation: str,
+    scope: str = "column",
+) -> dict[str, Any]:
+    return {
+        "code": code,
+        "title": title,
+        "severity": normalize_severity(severity),
+        "scope": scope,
+        "status": "Review",
+        "evidence": evidence,
+        "recommendation": recommendation,
+        "method": "quality_diagnostics",
+        "confidence": "deterministic",
+    }
+
+
+def findings_from_quality_diagnostics(
+    diagnostics: Mapping[str, Any] | None,
+) -> list[dict[str, Any]]:
+    """Translate deterministic quality diagnostics into actionable findings."""
+    if not isinstance(diagnostics, Mapping) or not diagnostics.get("available"):
+        return []
+
+    findings: list[dict[str, Any]] = []
+
+    for item in diagnostics.get("identifier_duplicates", []):
+        if not isinstance(item, Mapping):
+            continue
+        column = str(item.get("column") or "identifier")
+        duplicate_rows = int(item.get("duplicate_rows") or 0)
+        findings.append(_quality_finding(
+            code=f"quality.identifier_duplicates.{_slug(column)}",
+            title=f"Duplicate identifiers: {column}",
+            severity=item.get("severity", "high"),
+            evidence=f"{duplicate_rows} rows share duplicated values in identifier-like column '{column}'.",
+            recommendation=(
+                f"Verify uniqueness rules for '{column}' and resolve duplicated identifiers before joins, "
+                "deduplication, or model evaluation."
+            ),
+        ))
+
+    for item in diagnostics.get("duplicate_columns", []):
+        if not isinstance(item, Mapping):
+            continue
+        canonical = str(item.get("canonical_column") or "column")
+        duplicates = [str(value) for value in item.get("duplicate_columns", [])]
+        if not duplicates:
+            continue
+        findings.append(_quality_finding(
+            code=f"quality.duplicate_columns.{_slug(canonical)}",
+            title=f"Duplicate columns: {canonical}",
+            severity=item.get("severity", "medium"),
+            evidence=(
+                f"'{canonical}' is exactly duplicated by: {', '.join(duplicates)}."
+            ),
+            recommendation=(
+                "Remove or explicitly document redundant columns to reduce ambiguity, memory usage, and leakage risk."
+            ),
+        ))
+
+    for item in diagnostics.get("quasi_constant_columns", []):
+        if not isinstance(item, Mapping):
+            continue
+        column = str(item.get("column") or "column")
+        ratio = float(item.get("top_value_ratio") or 0)
+        findings.append(_quality_finding(
+            code=f"quality.quasi_constant.{_slug(column)}",
+            title=f"Quasi-constant column: {column}",
+            severity=item.get("severity", "low"),
+            evidence=f"One value represents approximately {ratio:.1%} of non-missing sampled values.",
+            recommendation=(
+                f"Review whether '{column}' carries useful signal; near-constant features often add little analytical value."
+            ),
+        ))
+
+    for item in diagnostics.get("coercion_candidates", []):
+        if not isinstance(item, Mapping):
+            continue
+        column = str(item.get("column") or "column")
+        suggested = str(item.get("suggested_type") or "structured")
+        ratio = float(item.get("parse_ratio") or 0)
+        findings.append(_quality_finding(
+            code=f"quality.coercion.{_slug(column)}",
+            title=f"Type coercion candidate: {column}",
+            severity=item.get("severity", "low"),
+            evidence=f"Approximately {ratio:.1%} of sampled values parse cleanly as {suggested}.",
+            recommendation=(
+                f"Consider converting '{column}' to {suggested} after reviewing non-parsing values and preserving intended semantics."
+            ),
+        ))
+
+    for item in diagnostics.get("category_normalisation", []):
+        if not isinstance(item, Mapping):
+            continue
+        column = str(item.get("column") or "column")
+        groups = int(item.get("variant_group_count") or 0)
+        findings.append(_quality_finding(
+            code=f"quality.category_normalisation.{_slug(column)}",
+            title=f"Category normalization issue: {column}",
+            severity=item.get("severity", "medium"),
+            evidence=f"Detected {groups} category groups that differ only by case or surrounding whitespace.",
+            recommendation=(
+                f"Normalize whitespace/casing in '{column}' with an explicit mapping before grouping, validation, or modelling."
+            ),
+        ))
+
+    for item in diagnostics.get("blank_strings", []):
+        if not isinstance(item, Mapping):
+            continue
+        column = str(item.get("column") or "column")
+        count = int(item.get("blank_count_in_sample") or 0)
+        findings.append(_quality_finding(
+            code=f"quality.blank_strings.{_slug(column)}",
+            title=f"Blank strings: {column}",
+            severity=item.get("severity", "medium"),
+            evidence=f"Found {count} blank/whitespace-only values in the diagnostic sample.",
+            recommendation=(
+                f"Treat blank strings in '{column}' consistently as missing values or a documented category."
+            ),
+        ))
+
+    for item in diagnostics.get("infinite_values", []):
+        if not isinstance(item, Mapping):
+            continue
+        column = str(item.get("column") or "column")
+        count = int(item.get("infinite_count_in_sample") or 0)
+        findings.append(_quality_finding(
+            code=f"quality.infinite_values.{_slug(column)}",
+            title=f"Infinite numeric values: {column}",
+            severity=item.get("severity", "high"),
+            evidence=f"Found {count} positive/negative infinite values in the diagnostic sample.",
+            recommendation=(
+                f"Replace or explicitly handle infinities in '{column}' before statistics, scaling, or model training."
+            ),
+        ))
+
+    for item in diagnostics.get("mixed_object_types", []):
+        if not isinstance(item, Mapping):
+            continue
+        column = str(item.get("column") or "column")
+        types = item.get("python_types", {})
+        findings.append(_quality_finding(
+            code=f"quality.mixed_object_types.{_slug(column)}",
+            title=f"Mixed Python types: {column}",
+            severity=item.get("severity", "medium"),
+            evidence=f"Object column contains multiple runtime value types: {types}.",
+            recommendation=(
+                f"Standardize the representation of '{column}' before serialization, joins, validation, or type conversion."
+            ),
+        ))
+
+    for item in diagnostics.get("missingness_relationships", []):
+        if not isinstance(item, Mapping):
+            continue
+        columns = [str(value) for value in item.get("columns", [])]
+        if len(columns) != 2:
+            continue
+        jaccard = float(item.get("jaccard") or 0)
+        findings.append(_quality_finding(
+            code=f"quality.missingness_relationship.{_slug(columns[0])}.{_slug(columns[1])}",
+            title=f"Linked missingness: {columns[0]} + {columns[1]}",
+            severity=item.get("severity", "low"),
+            evidence=f"Their missing-value masks have Jaccard similarity {jaccard:.2f} in the diagnostic sample.",
+            recommendation=(
+                "Investigate whether these columns are jointly missing because of one upstream process, segment, or collection rule."
+            ),
+            scope="dataset",
+        ))
+
+    return _sort_findings(findings)
+
+
 def merge_findings(*groups: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
     """Merge finding groups, de-duplicating by stable code."""
     merged: list[dict[str, Any]] = []

From 3f243fac5e3c78b4d549500ea34a071bc9e1ce46 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:04:10 +0530
Subject: [PATCH 075/439] feat: surface quality diagnostics through result
 findings

---
 src/framevitals/result.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/result.py b/src/framevitals/result.py
index 144cf85..683083e 100644
--- a/src/framevitals/result.py
+++ b/src/framevitals/result.py
@@ -13,6 +13,7 @@
 from typing import Any
 
 from framevitals.findings import (
+    findings_from_quality_diagnostics,
     findings_from_signals,
     findings_from_target_intelligence,
     merge_findings,
@@ -44,6 +45,7 @@ def __init__(self, *args, **kwargs) -> None:
                 signals = []
             self["findings"] = merge_findings(
                 findings_from_signals(signals),
+                findings_from_quality_diagnostics(self.get("quality_diagnostics")),
                 findings_from_target_intelligence(self.get("target_intelligence")),
             )
 
@@ -118,7 +120,7 @@ def summary(self) -> dict[str, Any]:
         }
 
     def column(self, name: str) -> ColumnResult:
-        """Return the combined profile, role, and semantic view for one column."""
+        """Return the combined profile, role, semantic, and quality view for one column."""
         profile = self.get("profile", {})
         roles = self.get("column_roles", {})
         if not isinstance(profile, dict):
@@ -144,6 +146,16 @@ def column(self, name: str) -> ColumnResult:
         if not isinstance(correlations, dict):
             correlations = {}
 
+        quality_findings = [
+            finding
+            for finding in self.findings
+            if str(finding.get("code", "")).startswith("quality.")
+            and (
+                f".{name.lower().replace(' ', '_')}" in str(finding.get("code", ""))
+                or name in str(finding.get("title", ""))
+            )
+        ]
+
         payload = {
             "name": name,
             "dtype": profile.get("dtypes", {}).get(name),
@@ -161,6 +173,7 @@ def column(self, name: str) -> ColumnResult:
             "numeric_summary": numeric_summary.get(name),
             "categorical_summary": categorical_summary.get(name),
             "correlations": correlations.get(name, {}),
+            "quality_findings": quality_findings,
         }
         return ColumnResult(payload)
 

From 74499205b3f03a7308344f6b245dff7587e92543 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:04:41 +0530
Subject: [PATCH 076/439] refactor: derive plan modules from central registry

---
 src/framevitals/api.py | 22 ++++------------------
 1 file changed, 4 insertions(+), 18 deletions(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index 4fea9d9..bd37ae9 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -14,7 +14,7 @@
     infer_cleaning_plan,
 )
 from framevitals.column_roles import infer_column_roles
-from framevitals.config import ConfigInput, resolve_config
+from framevitals.config import ConfigInput, VALID_MODULES, resolve_config
 from framevitals.contracts import infer_contract as _infer_contract
 from framevitals.contracts import validate_contract
 from framevitals.dataset_signals import detect_dataset_signals
@@ -151,24 +151,10 @@ def plan(
         analysis_mode=resolved.mode,
         target_column=resolved.target,
     )
+    disabled = set(resolved.disabled_modules)
     selection["execution_modules"] = {
-        "disabled": sorted(resolved.disabled_modules),
-        "enabled": sorted(
-            name
-            for name in (
-                "deep_statistics",
-                "anomaly_detection",
-                "time_series",
-                "text_profile",
-                "target_intelligence",
-                "modeling",
-                "explainability",
-                "cleaning",
-                "charts",
-                "ai",
-            )
-            if name not in resolved.disabled_modules
-        ),
+        "disabled": sorted(disabled),
+        "enabled": sorted(VALID_MODULES - disabled),
     }
 
     public_signals = {

From b4c368d20d213f8e3216bb58b89203aeda957bd1 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:05:01 +0530
Subject: [PATCH 077/439] test: include quality diagnostics in execution
 controls

---
 tests/test_execution_controls.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/test_execution_controls.py b/tests/test_execution_controls.py
index 5887d5f..a76685c 100644
--- a/tests/test_execution_controls.py
+++ b/tests/test_execution_controls.py
@@ -21,6 +21,7 @@ def test_analyze_can_disable_expensive_modules_without_losing_result_keys(tmp_pa
         mode="standard",
         artifacts=True,
         disabled_modules=[
+            "quality_diagnostics",
             "deep_statistics",
             "anomaly_detection",
             "time_series",
@@ -42,6 +43,7 @@ def test_analyze_can_disable_expensive_modules_without_losing_result_keys(tmp_pa
         for status in result["execution"]["module_status"].values()
     )
 
+    assert result["quality_diagnostics"]["skipped"] is True
     assert result["deep_statistics_v2"]["skipped"] is True
     assert result["anomalies_v2"]["skipped"] is True
     assert result["time_series"]["skipped"] is True
@@ -57,7 +59,7 @@ def test_analyze_can_disable_expensive_modules_without_losing_result_keys(tmp_pa
     assert not (tmp_path / "static" / "charts").exists()
 
 
-def test_ci_preset_skips_modeling_but_keeps_target_intelligence():
+def test_ci_preset_skips_modeling_but_keeps_target_intelligence_and_quality():
     result = framevitals.analyze(
         _frame(),
         target="target",
@@ -70,9 +72,11 @@ def test_ci_preset_skips_modeling_but_keeps_target_intelligence():
     assert execution["charts"] == "disabled"
     assert execution["ai"] == "disabled"
     assert execution["target_intelligence"] == "ran"
+    assert execution["quality_diagnostics"] == "ran"
 
     assert result["model_leaderboard"]["skipped"] is True
     assert result["target_intelligence"]["available"] is True
+    assert result["quality_diagnostics"]["available"] is True
 
 
 def test_plan_surfaces_execution_module_selection():
@@ -86,6 +90,7 @@ def test_plan_surfaces_execution_module_selection():
     modules = plan["selection"]["execution_modules"]
     assert modules["disabled"] == ["charts", "modeling"]
     assert "anomaly_detection" in modules["enabled"]
+    assert "quality_diagnostics" in modules["enabled"]
     assert plan["config"]["disabled_modules"] == ("modeling", "charts")
 
 

From 6ba4463b0e3bfb785dc9c90a9d200f44c6e2beb2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:05:24 +0530
Subject: [PATCH 078/439] test: cover practical quality diagnostics end to end

---
 tests/test_quality_diagnostics_v2.py | 144 +++++++++++++++++++++++++++
 1 file changed, 144 insertions(+)
 create mode 100644 tests/test_quality_diagnostics_v2.py

diff --git a/tests/test_quality_diagnostics_v2.py b/tests/test_quality_diagnostics_v2.py
new file mode 100644
index 0000000..e6aee74
--- /dev/null
+++ b/tests/test_quality_diagnostics_v2.py
@@ -0,0 +1,144 @@
+import numpy as np
+import pandas as pd
+
+import framevitals
+from framevitals.column_roles import infer_column_roles
+from framevitals.profiler import build_profile
+from framevitals.quality_diagnostics import run_quality_diagnostics
+
+
+def _messy_frame() -> pd.DataFrame:
+    rows = 40
+    missing_pattern = [None if index < 10 else index for index in range(rows)]
+    return pd.DataFrame({
+        "customer_id": [f"CUST-{index:03d}" for index in range(rows)],
+        "order_id": [f"ORD-{index // 2:03d}" for index in range(rows)],
+        "status": ["active"] * 38 + ["inactive"] * 2,
+        "base": list(range(rows)),
+        "base_copy": list(range(rows)),
+        "numeric_text": [str(index % 20) for index in range(rows)],
+        "event_text": [f"2026-01-{(index % 20) + 1:02d}" for index in range(rows)],
+        "city": ["Pune", " pune ", "PUNE", "Mumbai"] * 10,
+        "notes": ["", "ok", "  ", "ready"] * 10,
+        "ratio": [float(index) for index in range(rows - 1)] + [np.inf],
+        "mixed": ["1", 2, "3", 4] * 10,
+        "missing_a": missing_pattern,
+        "missing_b": list(missing_pattern),
+    })
+
+
+def test_quality_diagnostics_detect_real_world_data_problems():
+    df = _messy_frame()
+    profile = build_profile(df)
+    roles = infer_column_roles(df)
+
+    result = run_quality_diagnostics(
+        df,
+        profile=profile,
+        column_roles=roles,
+    )
+
+    assert result["available"] is True
+    assert result["summary"]["issue_count"] > 0
+
+    key_columns = {item["column"] for item in result["primary_key_candidates"]}
+    assert "customer_id" in key_columns
+
+    duplicate_id_columns = {
+        item["column"] for item in result["identifier_duplicates"]
+    }
+    assert "order_id" in duplicate_id_columns
+
+    quasi_columns = {item["column"] for item in result["quasi_constant_columns"]}
+    assert "status" in quasi_columns
+
+    duplicate_groups = result["duplicate_columns"]
+    assert any(
+        item["canonical_column"] == "base"
+        and "base_copy" in item["duplicate_columns"]
+        for item in duplicate_groups
+    )
+
+    coercions = {
+        (item["column"], item["suggested_type"])
+        for item in result["coercion_candidates"]
+    }
+    assert ("numeric_text", "numeric") in coercions
+    assert ("event_text", "datetime") in coercions
+
+    normalized_columns = {
+        item["column"] for item in result["category_normalisation"]
+    }
+    assert "city" in normalized_columns
+
+    blank_columns = {item["column"] for item in result["blank_strings"]}
+    assert "notes" in blank_columns
+
+    infinity_columns = {item["column"] for item in result["infinite_values"]}
+    assert "ratio" in infinity_columns
+
+    mixed_columns = {item["column"] for item in result["mixed_object_types"]}
+    assert "mixed" in mixed_columns
+
+    pairs = {
+        tuple(item["columns"])
+        for item in result["missingness_relationships"]
+    }
+    assert ("missing_a", "missing_b") in pairs
+
+
+def test_quality_diagnostics_are_bounded_and_report_column_truncation():
+    df = pd.DataFrame({
+        f"col_{index}": list(range(100))
+        for index in range(8)
+    })
+
+    result = run_quality_diagnostics(
+        df,
+        max_sample_rows=20,
+        max_columns=3,
+    )
+
+    assert result["columns_checked"] == 3
+    assert result["truncated_columns"] is True
+    assert result["max_sample_rows"] == 20
+
+
+def test_public_analysis_surfaces_quality_diagnostics_as_findings():
+    result = framevitals.analyze(
+        _messy_frame(),
+        mode="quick",
+        disabled_modules=["cleaning", "ai"],
+    )
+
+    diagnostics = result["quality_diagnostics"]
+    assert diagnostics["available"] is True
+    assert result["execution"]["module_status"]["quality_diagnostics"] == "ran"
+    assert "quality_diagnostics" in result["timings_ms"]
+
+    quality_findings = [
+        finding
+        for finding in result.findings
+        if finding["code"].startswith("quality.")
+    ]
+    codes = {finding["code"] for finding in quality_findings}
+
+    assert any(code.startswith("quality.identifier_duplicates.order_id") for code in codes)
+    assert any(code.startswith("quality.duplicate_columns.base") for code in codes)
+    assert any(code.startswith("quality.infinite_values.ratio") for code in codes)
+    assert result.column("city")["quality_findings"]
+
+
+def test_quality_diagnostics_can_be_disabled_without_removing_result_key():
+    result = framevitals.analyze(
+        _messy_frame(),
+        mode="quick",
+        disabled_modules=["quality_diagnostics", "cleaning", "ai"],
+    )
+
+    assert result["quality_diagnostics"]["skipped"] is True
+    assert result["execution"]["module_status"]["quality_diagnostics"] == "disabled"
+    assert not any(
+        finding["code"].startswith("quality.")
+        for finding in result.findings
+    )

From cf3acb85e1e5cee1c46e276348f02ae9ab35d3e1 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:07:43 +0530
Subject: [PATCH 079/439] feat: make anomaly ensemble safer and more
 explainable

---
 src/framevitals/anomaly_ensemble.py | 419 ++++++++++++++++------------
 1 file changed, 248 insertions(+), 171 deletions(-)

diff --git a/src/framevitals/anomaly_ensemble.py b/src/framevitals/anomaly_ensemble.py
index 30dfebe..87af125 100644
--- a/src/framevitals/anomaly_ensemble.py
+++ b/src/framevitals/anomaly_ensemble.py
@@ -1,28 +1,17 @@
-"""
-Anomaly Ensemble (WS-2)
-=======================
-Multi-detector anomaly scoring for tabular data.
-
-Detectors:
-    1. IsolationForest          (sklearn.ensemble)
-    2. LocalOutlierFactor       (sklearn.neighbors)
-    3. EllipticEnvelope         (sklearn.covariance) — robust Gaussian
-    4. Robust z-score (MAD)     (numpy)
-    5. Mahalanobis distance     (sklearn shrinkage covariance)
-    6. ECOD                     (pyod, optional)
-    7. COPOD                    (pyod, optional)
-
-Each detector emits a normalized score in [0, 1] (higher = more anomalous).
-The ensemble score is the column-wise mean across available detectors.
-
-Public entry point:
-    detect_anomalies_ensemble(df, contamination=0.05) -> dict
+"""Multi-detector anomaly scoring for tabular data.
+
+The ensemble combines deterministic/classical detectors that are useful across
+many tabular datasets and optionally adds ECOD/COPOD when PyOD is installed.
+Scores remain normalized to [0, 1] and the historical mean-ensemble threshold
+is preserved, while the result now also reports detector agreement, failures,
+input preparation, and feature-level context for top anomalous rows.
 """
 
 from __future__ import annotations
 
+import math
 import warnings
-from typing import Any
+from typing import Any, Callable
 
 import numpy as np
 import pandas as pd
@@ -34,10 +23,6 @@
 from sklearn.preprocessing import StandardScaler
 
 
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
 def _to_unit(scores: np.ndarray) -> np.ndarray:
     """Min-max normalize an arbitrary score vector to [0, 1]."""
     s = np.asarray(scores, dtype=float)
@@ -49,44 +34,84 @@ def _to_unit(scores: np.ndarray) -> np.ndarray:
     if hi - lo < 1e-12:
         return np.zeros_like(s)
     out = (s - lo) / (hi - lo)
-    out = np.where(np.isnan(out), 0.0, out)
-    return out
+    return np.where(np.isnan(out), 0.0, out)
 
 
-def _prepare_numeric_matrix(df: pd.DataFrame, max_columns: int = 30):
-    """
-    Select numeric columns, drop constants, impute medians, scale.
-    Returns (X_scaled_df, used_columns) or (None, []) if not enough columns.
-    """
+def _prepare_numeric_matrix(
+    df: pd.DataFrame,
+    max_columns: int = 30,
+) -> tuple[pd.DataFrame | None, list[str], dict[str, Any]]:
+    """Select, sanitize, impute, and scale numeric columns."""
     numeric = df.select_dtypes(include=[np.number]).copy()
-
-    # Drop constants
-    for col in list(numeric.columns):
-        if numeric[col].nunique(dropna=True) <= 1:
-            numeric = numeric.drop(columns=[col])
+    original_numeric_columns = list(numeric.columns)
+
+    infinity_counts: dict[str, int] = {}
+    for column in list(numeric.columns):
+        values = pd.to_numeric(numeric[column], errors="coerce")
+        finite_array = values.to_numpy(dtype="float64", na_value=np.nan)
+        inf_count = int(np.isinf(finite_array).sum())
+        if inf_count:
+            infinity_counts[column] = inf_count
+        numeric[column] = values.replace([np.inf, -np.inf], np.nan)
+
+    dropped_all_missing: list[str] = []
+    dropped_constant: list[str] = []
+    for column in list(numeric.columns):
+        if numeric[column].notna().sum() == 0:
+            numeric = numeric.drop(columns=[column])
+            dropped_all_missing.append(column)
+        elif numeric[column].nunique(dropna=True) <= 1:
+            numeric = numeric.drop(columns=[column])
+            dropped_constant.append(column)
 
     if numeric.shape[1] < 1:
-        return None, []
+        metadata = {
+            "numeric_columns_found": len(original_numeric_columns),
+            "used_columns": [],
+            "dropped_constant_columns": dropped_constant,
+            "dropped_all_missing_columns": dropped_all_missing,
+            "infinite_values_replaced": infinity_counts,
+            "missing_values_imputed": {},
+            "truncated_columns": False,
+        }
+        return None, [], metadata
 
-    # Cap dimensionality (keep most-populated columns)
-    if numeric.shape[1] > max_columns:
+    truncated = numeric.shape[1] > max_columns
+    if truncated:
         keep = (
-            numeric.notna().sum().sort_values(ascending=False).head(max_columns).index.tolist()
+            numeric.notna()
+            .sum()
+            .sort_values(ascending=False)
+            .head(max_columns)
+            .index.tolist()
         )
         numeric = numeric[keep]
 
-    # Median impute then scale
+    missing_counts = {
+        column: int(count)
+        for column, count in numeric.isna().sum().items()
+        if int(count) > 0
+    }
+
     imputer = SimpleImputer(strategy="median")
     scaler = StandardScaler()
     X = imputer.fit_transform(numeric)
     X = scaler.fit_transform(X)
 
-    return pd.DataFrame(X, columns=numeric.columns, index=df.index), list(numeric.columns)
-
+    used_columns = list(numeric.columns)
+    metadata = {
+        "numeric_columns_found": len(original_numeric_columns),
+        "used_columns": used_columns,
+        "dropped_constant_columns": dropped_constant,
+        "dropped_all_missing_columns": dropped_all_missing,
+        "infinite_values_replaced": {
+            key: value for key, value in infinity_counts.items() if key in used_columns
+        },
+        "missing_values_imputed": missing_counts,
+        "truncated_columns": truncated,
+    }
+    return pd.DataFrame(X, columns=used_columns, index=df.index), used_columns, metadata
 
-# ---------------------------------------------------------------------------
-# Individual detectors
-# ---------------------------------------------------------------------------
 
 def _detect_iforest(X: np.ndarray, contamination: float) -> np.ndarray:
     model = IsolationForest(
@@ -96,26 +121,25 @@ def _detect_iforest(X: np.ndarray, contamination: float) -> np.ndarray:
         n_jobs=-1,
     )
     model.fit(X)
-    # score_samples: higher = more normal. Negate so higher = more anomalous.
-    raw = -model.score_samples(X)
-    return _to_unit(raw)
+    return _to_unit(-model.score_samples(X))
 
 
 def _detect_lof(X: np.ndarray) -> np.ndarray:
-    model = LocalOutlierFactor(n_neighbors=min(20, max(5, X.shape[0] // 20)), n_jobs=-1)
+    n_neighbors = min(20, max(5, X.shape[0] // 20))
+    n_neighbors = min(n_neighbors, max(X.shape[0] - 1, 1))
+    model = LocalOutlierFactor(n_neighbors=n_neighbors, n_jobs=-1)
     model.fit_predict(X)
-    raw = -model.negative_outlier_factor_  # higher = more anomalous
-    return _to_unit(raw)
+    return _to_unit(-model.negative_outlier_factor_)
 
 
 def _detect_elliptic(X: np.ndarray, contamination: float) -> np.ndarray:
-    try:
-        model = EllipticEnvelope(contamination=contamination, support_fraction=None, random_state=42)
-        model.fit(X)
-        raw = -model.score_samples(X)
-        return _to_unit(raw)
-    except Exception:
-        return np.zeros(X.shape[0])
+    model = EllipticEnvelope(
+        contamination=contamination,
+        support_fraction=None,
+        random_state=42,
+    )
+    model.fit(X)
+    return _to_unit(-model.score_samples(X))
 
 
 def _detect_mad_robust_z(X: np.ndarray) -> np.ndarray:
@@ -124,50 +148,70 @@ def _detect_mad_robust_z(X: np.ndarray) -> np.ndarray:
     mad = np.median(np.abs(X - median), axis=0)
     mad_safe = np.where(mad > 0, mad, 1.0)
     z = np.abs((X - median) / (1.4826 * mad_safe))
-    raw = z.mean(axis=1)
-    return _to_unit(raw)
+    return _to_unit(z.mean(axis=1))
 
 
 def _detect_mahalanobis(X: np.ndarray) -> np.ndarray:
-    """Mahalanobis distance using a shrinkage / robust covariance estimate."""
+    """Mahalanobis distance using robust covariance with a stable fallback."""
     try:
         cov = MinCovDet(random_state=42).fit(X)
         raw = cov.mahalanobis(X)
         return _to_unit(raw)
     except Exception:
-        # Fallback to plain covariance
-        try:
-            mean = X.mean(axis=0)
-            cov_matrix = np.cov(X.T) + np.eye(X.shape[1]) * 1e-6
-            inv = np.linalg.pinv(cov_matrix)
-            diff = X - mean
-            raw = np.einsum("ij,jk,ik->i", diff, inv, diff)
-            return _to_unit(np.sqrt(np.maximum(raw, 0)))
-        except Exception:
-            return np.zeros(X.shape[0])
-
-
-def _detect_pyod(X: np.ndarray, model_name: str) -> np.ndarray | None:
-    """Try ECOD or COPOD from pyod; return None if pyod is not installed."""
-    try:
-        if model_name == "ecod":
-            from pyod.models.ecod import ECOD as Model
-        elif model_name == "copod":
-            from pyod.models.copod import COPOD as Model
-        else:
-            return None
-        model = Model()
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            model.fit(X)
-        return _to_unit(model.decision_scores_)
-    except Exception:
-        return None
+        mean = X.mean(axis=0)
+        cov_matrix = np.atleast_2d(np.cov(X.T))
+        cov_matrix = cov_matrix + np.eye(X.shape[1]) * 1e-6
+        inv = np.linalg.pinv(cov_matrix)
+        diff = X - mean
+        raw = np.einsum("ij,jk,ik->i", diff, inv, diff)
+        return _to_unit(np.sqrt(np.maximum(raw, 0)))
+
+
+def _detect_pyod(X: np.ndarray, model_name: str) -> np.ndarray:
+    if model_name == "ecod":
+        from pyod.models.ecod import ECOD as Model
+    elif model_name == "copod":
+        from pyod.models.copod import COPOD as Model
+    else:
+        raise ValueError(f"Unsupported PyOD detector: {model_name}")
+
+    model = Model()
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        model.fit(X)
+    return _to_unit(model.decision_scores_)
 
 
-# ---------------------------------------------------------------------------
-# Public entry point
-# ---------------------------------------------------------------------------
+def _detector_summary(scores: np.ndarray) -> dict[str, float]:
+    values = np.asarray(scores, dtype=float)
+    return {
+        "mean": round(float(np.mean(values)), 4),
+        "median": round(float(np.median(values)), 4),
+        "p95": round(float(np.quantile(values, 0.95)), 4),
+        "p99": round(float(np.quantile(values, 0.99)), 4),
+        "max": round(float(np.max(values)), 4),
+    }
+
+
+def _top_feature_deviations(
+    X_df: pd.DataFrame,
+    row_index: Any,
+    *,
+    limit: int = 3,
+) -> list[dict[str, Any]]:
+    row = X_df.loc[row_index]
+    if isinstance(row, pd.DataFrame):
+        row = row.iloc[0]
+    ordered = row.abs().sort_values(ascending=False).head(limit)
+    return [
+        {
+            "feature": str(feature),
+            "standardized_deviation": round(float(abs(row[feature])), 4),
+            "direction": "high" if float(row[feature]) >= 0 else "low",
+        }
+        for feature in ordered.index
+    ]
+
 
 def detect_anomalies_ensemble(
     df: pd.DataFrame,
@@ -175,35 +219,25 @@ def detect_anomalies_ensemble(
     threshold: float = 0.6,
     max_columns: int = 30,
     top_k: int = 25,
-) -> dict:
-    """
-    Run all available detectors and return a JSON-safe summary.
-
-    Args:
-        df: Input dataframe.
-        contamination: Expected anomaly fraction passed to detectors that need it.
-        threshold: Ensemble score above which a row is flagged.
-        max_columns: Cap on numeric dimensionality.
-        top_k: Number of top rows to include in the output payload.
-
-    Returns shape:
-        {
-            "available": bool,
-            "n_rows_scored": int,
-            "used_columns": [...],
-            "detectors_run": [...],
-            "threshold": float,
-            "flagged_count": int,
-            "ensemble_summary": {"min","mean","median","max","p95","p99"},
-            "top_rows": [{"row_index", "ensemble", "<detector>": score, ...}],
-            "score_table": pandas-records (top_k rows only),
-        }
-    """
-    X_df, used_cols = _prepare_numeric_matrix(df, max_columns=max_columns)
+) -> dict[str, Any]:
+    """Run available anomaly detectors and return an explainable JSON-safe summary."""
+    if max_columns < 1:
+        raise ValueError("max_columns must be at least 1.")
+    if top_k < 1:
+        raise ValueError("top_k must be at least 1.")
+    if not 0 <= threshold <= 1:
+        raise ValueError("threshold must be between 0 and 1.")
+
+    contamination = float(np.clip(contamination, 0.001, 0.5))
+    X_df, used_cols, preparation = _prepare_numeric_matrix(
+        df,
+        max_columns=max_columns,
+    )
     if X_df is None or X_df.empty:
         return {
             "available": False,
             "message": "No usable numeric columns (need at least 1 non-constant numeric column).",
+            "preparation": preparation,
         }
 
     X = X_df.values
@@ -213,72 +247,89 @@ def detect_anomalies_ensemble(
             "available": False,
             "message": "Need at least 20 rows for anomaly ensemble.",
             "used_columns": used_cols,
+            "preparation": preparation,
         }
 
-    # Bound contamination
-    contamination = float(np.clip(contamination, 0.001, 0.5))
-
     detector_scores: dict[str, np.ndarray] = {}
+    detectors_failed: dict[str, str] = {}
+    detectors_skipped: dict[str, str] = {}
 
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-
-        try:
-            detector_scores["isolation_forest"] = _detect_iforest(X, contamination)
-        except Exception:
-            pass
-
-        try:
-            detector_scores["local_outlier_factor"] = _detect_lof(X)
-        except Exception:
-            pass
-
-        try:
-            elliptic = _detect_elliptic(X, contamination)
-            if elliptic.any():
-                detector_scores["elliptic_envelope"] = elliptic
-        except Exception:
-            pass
-
+    def run_detector(name: str, fn: Callable[[], np.ndarray]) -> None:
         try:
-            detector_scores["mad_robust_z"] = _detect_mad_robust_z(X)
-        except Exception:
-            pass
+            scores = np.asarray(fn(), dtype=float)
+            if scores.shape != (n_rows,):
+                raise ValueError(
+                    f"Detector returned shape {scores.shape}; expected {(n_rows,)}."
+                )
+            if not np.isfinite(scores).all():
+                raise ValueError("Detector produced non-finite scores.")
+            detector_scores[name] = scores
+        except ImportError:
+            detectors_skipped[name] = "Optional dependency is not installed."
+        except Exception as exc:  # noqa: BLE001 - individual detectors fail soft
+            detectors_failed[name] = f"{type(exc).__name__}: {exc}"
 
-        try:
-            mahal = _detect_mahalanobis(X)
-            if mahal.any():
-                detector_scores["mahalanobis"] = mahal
-        except Exception:
-            pass
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        run_detector(
+            "isolation_forest",
+            lambda: _detect_iforest(X, contamination),
+        )
+        run_detector("local_outlier_factor", lambda: _detect_lof(X))
 
-        ecod = _detect_pyod(X, "ecod")
-        if ecod is not None:
-            detector_scores["ecod"] = ecod
+        if n_rows >= max(20, X.shape[1] * 2 + 1):
+            run_detector(
+                "elliptic_envelope",
+                lambda: _detect_elliptic(X, contamination),
+            )
+        else:
+            detectors_skipped["elliptic_envelope"] = (
+                "Too few rows relative to numeric dimensionality for stable covariance estimation."
+            )
 
-        copod = _detect_pyod(X, "copod")
-        if copod is not None:
-            detector_scores["copod"] = copod
+        run_detector("mad_robust_z", lambda: _detect_mad_robust_z(X))
+        run_detector("mahalanobis", lambda: _detect_mahalanobis(X))
+        run_detector("ecod", lambda: _detect_pyod(X, "ecod"))
+        run_detector("copod", lambda: _detect_pyod(X, "copod"))
 
     if not detector_scores:
         return {
             "available": False,
-            "message": "All detectors failed.",
+            "message": "All anomaly detectors failed or were unavailable.",
             "used_columns": used_cols,
+            "preparation": preparation,
+            "detectors_failed": detectors_failed,
+            "detectors_skipped": detectors_skipped,
         }
 
-    # Stack detector scores into a (n_rows, n_detectors) matrix and average
-    score_matrix = np.column_stack(list(detector_scores.values()))
+    detector_names = list(detector_scores)
+    score_matrix = np.column_stack([detector_scores[name] for name in detector_names])
     ensemble = score_matrix.mean(axis=1)
 
-    # Build per-row table (in original df index)
     score_df = pd.DataFrame(
-        {name: scores for name, scores in detector_scores.items()},
+        {name: detector_scores[name] for name in detector_names},
         index=df.index,
     )
     score_df["ensemble"] = ensemble
 
-    # Summary stats
+    vote_thresholds: dict[str, float] = {}
+    vote_matrix: list[np.ndarray] = []
+    for name in detector_names:
+        detector_threshold = float(
+            np.quantile(detector_scores[name], 1.0 - contamination)
+        )
+        vote_thresholds[name] = detector_threshold
+        vote_matrix.append(detector_scores[name] >= detector_threshold)
+
+    agreement_count = np.column_stack(vote_matrix).sum(axis=1)
+    agreement_fraction = agreement_count / len(detector_names)
+    score_df["agreement_count"] = agreement_count
+    score_df["agreement_fraction"] = agreement_fraction
+
+    majority_required = max(1, math.ceil(len(detector_names) / 2))
+    consensus_mask = agreement_count >= majority_required
+    flagged_mask = ensemble >= threshold
+
     es = ensemble[np.isfinite(ensemble)]
     summary = {
         "min": float(np.min(es)),
@@ -289,32 +340,58 @@ def detect_anomalies_ensemble(
         "p99": float(np.quantile(es, 0.99)),
     }
 
-    flagged_mask = score_df["ensemble"] >= threshold
     flagged_count = int(flagged_mask.sum())
+    consensus_flagged_count = int(consensus_mask.sum())
 
-    # Top-k rows
     top = score_df.sort_values("ensemble", ascending=False).head(top_k)
-    top_rows = []
+    top_rows: list[dict[str, Any]] = []
     for idx, row in top.iterrows():
-        entry = {"row_index": int(idx) if isinstance(idx, (int, np.integer)) else str(idx)}
-        for name in detector_scores.keys():
+        entry: dict[str, Any] = {
+            "row_index": int(idx) if isinstance(idx, (int, np.integer)) else str(idx),
+        }
+        for name in detector_names:
             entry[name] = round(float(row[name]), 4)
         entry["ensemble"] = round(float(row["ensemble"]), 4)
+        entry["flagged"] = bool(row["ensemble"] >= threshold)
+        entry["agreement_count"] = int(row["agreement_count"])
+        entry["agreement_fraction"] = round(float(row["agreement_fraction"]), 4)
+        entry["top_feature_deviations"] = _top_feature_deviations(X_df, idx)
         top_rows.append(entry)
 
+    detector_summaries = {
+        name: _detector_summary(scores)
+        for name, scores in detector_scores.items()
+    }
+
     return {
         "available": True,
         "n_rows_scored": int(n_rows),
         "used_columns": used_cols,
-        "detectors_run": list(detector_scores.keys()),
+        "preparation": preparation,
+        "detectors_run": detector_names,
+        "detectors_failed": detectors_failed,
+        "detectors_skipped": detectors_skipped,
+        "detector_summaries": detector_summaries,
+        "detector_vote_thresholds": {
+            name: round(value, 4) for name, value in vote_thresholds.items()
+        },
         "threshold": float(threshold),
         "contamination": float(contamination),
+        "expected_anomaly_count": int(math.ceil(n_rows * contamination)),
         "flagged_count": flagged_count,
+        "flagged_fraction": round(float(flagged_count / n_rows), 4),
+        "consensus": {
+            "majority_detectors_required": majority_required,
+            "flagged_count": consensus_flagged_count,
+            "flagged_fraction": round(float(consensus_flagged_count / n_rows), 4),
+        },
         "ensemble_summary": {k: round(v, 4) for k, v in summary.items()},
         "top_rows": top_rows,
         "interpretation": (
-            "Each detector emits a [0,1] anomaly score; the ensemble is the per-row mean "
-            "across all available detectors. Rows with ensemble >= "
-            f"{threshold} are flagged."
+            "Each available detector emits a normalized [0,1] anomaly score; the historical "
+            "ensemble remains their per-row mean. The configured threshold controls the main "
+            "flag, while detector agreement independently reports how many detectors place a row "
+            "in their contamination-adjusted anomaly tail. Top feature deviations are standardized "
+            "context, not causal explanations."
         ),
     }

From 6cc8ee2410dae92742fb58bbcb449b400f098d8f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:08:07 +0530
Subject: [PATCH 080/439] test: cover safer explainable anomaly diagnostics

---
 tests/test_anomaly_diagnostics_v2.py | 80 ++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 tests/test_anomaly_diagnostics_v2.py

diff --git a/tests/test_anomaly_diagnostics_v2.py b/tests/test_anomaly_diagnostics_v2.py
new file mode 100644
index 0000000..f9dbef1
--- /dev/null
+++ b/tests/test_anomaly_diagnostics_v2.py
@@ -0,0 +1,80 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from framevitals.anomaly_ensemble import detect_anomalies_ensemble
+
+
+def _anomaly_frame() -> pd.DataFrame:
+    rng = np.random.default_rng(42)
+    rows = 80
+    x = rng.normal(0, 1, rows)
+    y = rng.normal(0, 1, rows)
+    x[-1] = 12.0
+    y[-1] = -10.0
+    x[5] = np.inf
+    y[7] = np.nan
+    return pd.DataFrame({"x": x, "y": y})
+
+
+def test_anomaly_ensemble_handles_infinity_and_reports_preparation():
+    result = detect_anomalies_ensemble(_anomaly_frame(), top_k=5)
+
+    assert result["available"] is True
+    assert result["n_rows_scored"] == 80
+    assert result["preparation"]["infinite_values_replaced"]["x"] == 1
+    assert result["preparation"]["missing_values_imputed"]["x"] == 1
+    assert result["preparation"]["missing_values_imputed"]["y"] == 1
+    assert result["detectors_run"]
+    assert isinstance(result["detectors_failed"], dict)
+    assert isinstance(result["detectors_skipped"], dict)
+
+
+def test_anomaly_ensemble_reports_detector_agreement_and_feature_context():
+    result = detect_anomalies_ensemble(
+        _anomaly_frame(),
+        contamination=0.05,
+        threshold=0.5,
+        top_k=5,
+    )
+
+    assert result["available"] is True
+    assert result["consensus"]["majority_detectors_required"] >= 1
+    assert 0 <= result["consensus"]["flagged_fraction"] <= 1
+    assert 0 <= result["flagged_fraction"] <= 1
+    assert result["expected_anomaly_count"] == 4
+
+    top = result["top_rows"][0]
+    assert "agreement_count" in top
+    assert "agreement_fraction" in top
+    assert "flagged" in top
+    assert top["top_feature_deviations"]
+    assert {
+        item["feature"] for item in top["top_feature_deviations"]
+    }.issubset({"x", "y"})
+
+    for detector in result["detectors_run"]:
+        assert detector in result["detector_summaries"]
+        assert detector in result["detector_vote_thresholds"]
+
+
+def test_anomaly_ensemble_keeps_historical_score_fields():
+    result = detect_anomalies_ensemble(_anomaly_frame())
+
+    assert result["threshold"] == 0.6
+    assert result["contamination"] == 0.05
+    assert "flagged_count" in result
+    assert "ensemble_summary" in result
+    assert "top_rows" in result
+    assert all("ensemble" in row for row in result["top_rows"])
+
+
+def test_anomaly_ensemble_validates_public_controls():
+    frame = _anomaly_frame()
+
+    with pytest.raises(ValueError, match="threshold"):
+        detect_anomalies_ensemble(frame, threshold=1.1)
+    with pytest.raises(ValueError, match="top_k"):
+        detect_anomalies_ensemble(frame, top_k=0)
+    with pytest.raises(ValueError, match="max_columns"):
+        detect_anomalies_ensemble(frame, max_columns=0)

From e1a9433a9551b4131043e24bde37a2788504fed7 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:09:51 +0530
Subject: [PATCH 081/439] fix: harden shared ML preprocessing and feature
 filtering

---
 src/framevitals/ml_preprocessing.py | 209 ++++++++++++++++++++++------
 1 file changed, 165 insertions(+), 44 deletions(-)

diff --git a/src/framevitals/ml_preprocessing.py b/src/framevitals/ml_preprocessing.py
index 1b56d09..f1dd4d1 100644
--- a/src/framevitals/ml_preprocessing.py
+++ b/src/framevitals/ml_preprocessing.py
@@ -1,9 +1,14 @@
+"""Unified ML preprocessing used by every FrameVitals model workflow.
+
+The module deliberately keeps feature selection conservative: obvious IDs,
+unsupported temporal columns, constants, and dangerously high-cardinality
+categoricals are excluded before sklearn preprocessing. Numeric infinities are
+converted to missing values so the standard median imputer can handle them.
 """
-Unified ML Preprocessing
-=========================
-Single source of truth for preparing features and target.
-ALL ML modules MUST call prepare_ml_matrix() instead of their own prepare_xy().
-"""
+
+from __future__ import annotations
+
+import re
 
 import numpy as np
 import pandas as pd
@@ -12,25 +17,79 @@
 from sklearn.pipeline import Pipeline
 from sklearn.preprocessing import FunctionTransformer, OneHotEncoder, StandardScaler
 
-_ID_KEYWORDS = [
-    "id", "uuid", "hash", "key", "identifier", "index",
-    "roll", "roll_number", "rollno", "roll number",
-    "application", "registration", "serial", "ticket",
-    "account", "customer_id", "order_id", "transaction_id",
-    "userid", "studentid", "student_id", "user_id",
-]
-
-_TIME_KEYWORDS = [
-    "time", "date", "timestamp", "created", "updated",
-    "datetime", "ts", "event",
-]
 
 _CATEGORICAL_DTYPES = ["object", "string", "category", "bool"]
+_IDENTIFIER_TOKENS = {"id", "uuid", "hash", "identifier", "index"}
+_IDENTIFIER_EXACT = {
+    "key",
+    "roll",
+    "roll_number",
+    "rollno",
+    "application",
+    "registration",
+    "serial",
+    "ticket",
+    "account",
+    "userid",
+    "studentid",
+}
+_TIME_TOKENS = {"time", "date", "timestamp", "datetime"}
+
+
+def _normalise_column_name(name: object) -> tuple[str, set[str]]:
+    normalized = re.sub(r"[^a-z0-9]+", "_", str(name).strip().lower()).strip("_")
+    tokens = {token for token in normalized.split("_") if token}
+    return normalized, tokens
+
+
+def _looks_like_identifier_name(name: object) -> bool:
+    """Detect identifier-like names without unsafe substring matching.
+
+    The old ``"id" in column_name`` approach incorrectly classified ordinary
+    names such as ``paid_amount``. Token/boundary matching keeps common ID
+    conventions while avoiding those false positives.
+    """
+    normalized, tokens = _normalise_column_name(name)
+    if normalized in _IDENTIFIER_EXACT:
+        return True
+    if tokens & _IDENTIFIER_TOKENS:
+        return True
+    if normalized.endswith("_key") or normalized.endswith("_number"):
+        prefix = normalized.rsplit("_", 1)[0]
+        if prefix in {"account", "serial", "ticket", "roll", "registration"}:
+            return True
+    return False
+
+
+def _looks_like_time_name(name: object) -> bool:
+    normalized, tokens = _normalise_column_name(name)
+    if tokens & _TIME_TOKENS:
+        return True
+    if normalized in {"created", "updated", "created_at", "updated_at"}:
+        return True
+    if normalized.endswith("_at") and tokens & {"created", "updated"}:
+        return True
+    return False
+
+
+def _is_categorical_dtype(series: pd.Series) -> bool:
+    return bool(
+        pd.api.types.is_object_dtype(series)
+        or pd.api.types.is_string_dtype(series.dtype)
+        or isinstance(series.dtype, pd.CategoricalDtype)
+        or pd.api.types.is_bool_dtype(series)
+    )
 
 
-def prepare_ml_matrix(df, target, drop_high_unique_ratio=0.95, min_non_missing=5):
-    """Prepare a clean feature matrix and target vector for ML."""
-    warnings = []
+def prepare_ml_matrix(
+    df,
+    target,
+    drop_high_unique_ratio=0.95,
+    min_non_missing=5,
+    max_categorical_levels=200,
+):
+    """Prepare a conservative feature matrix and target vector for ML."""
+    warnings: list[str] = []
 
     if target not in df.columns:
         return {
@@ -40,49 +99,108 @@ def prepare_ml_matrix(df, target, drop_high_unique_ratio=0.95, min_non_missing=5
             "categorical_features": [],
             "dropped_columns": [],
             "warnings": [f"Target '{target}' not found."],
+            "infinite_values_replaced": {},
             "usable": False,
         }
 
-    data = df.dropna(subset=[target]).copy()
+    if not 0 < drop_high_unique_ratio <= 1:
+        raise ValueError("drop_high_unique_ratio must be in (0, 1].")
+    if min_non_missing < 1:
+        raise ValueError("min_non_missing must be at least 1.")
+    if max_categorical_levels < 2:
+        raise ValueError("max_categorical_levels must be at least 2.")
+
+    data = df.copy()
+
+    target_infinite_count = 0
+    if pd.api.types.is_numeric_dtype(data[target]):
+        target_values = pd.to_numeric(data[target], errors="coerce")
+        target_array = target_values.to_numpy(dtype="float64", na_value=np.nan)
+        target_infinite_count = int(np.isinf(target_array).sum())
+        if target_infinite_count:
+            data[target] = target_values.replace([np.inf, -np.inf], np.nan)
+            warnings.append(
+                f"Treated {target_infinite_count} infinite target values as missing."
+            )
+
+    data = data.dropna(subset=[target]).copy()
     rows_dropped = len(df) - len(data)
     if rows_dropped > 0:
-        warnings.append(f"Dropped {rows_dropped} rows with missing target values.")
+        warnings.append(f"Dropped {rows_dropped} rows with missing/invalid target values.")
 
     y = data[target]
-    X = data.drop(columns=[target])
+    X = data.drop(columns=[target]).copy()
+
+    dropped: list[dict[str, str]] = []
+    dropped_names: set[str] = set()
+    infinite_values_replaced: dict[str, int] = {}
+
+    def drop(column: str, reason: str) -> None:
+        if column not in dropped_names:
+            dropped.append({"column": column, "reason": reason})
+            dropped_names.add(column)
 
-    dropped = []
-    rows = max(len(X), 1)
+    for column in list(X.columns):
+        series = X[column]
+        if pd.api.types.is_numeric_dtype(series):
+            numeric = pd.to_numeric(series, errors="coerce")
+            values = numeric.to_numpy(dtype="float64", na_value=np.nan)
+            inf_count = int(np.isinf(values).sum())
+            if inf_count:
+                X[column] = numeric.replace([np.inf, -np.inf], np.nan)
+                infinite_values_replaced[column] = inf_count
+                warnings.append(
+                    f"Replaced {inf_count} infinite values in '{column}' with missing values for imputation."
+                )
 
-    for col in list(X.columns):
-        non_missing = int(X[col].notna().sum())
-        unique_count = X[col].nunique(dropna=True)
-        unique_ratio = unique_count / rows
-        lower = col.lower().replace("-", "_")
+    for column in list(X.columns):
+        series = X[column]
+        non_missing = int(series.notna().sum())
+        unique_count = int(series.nunique(dropna=True))
+        unique_ratio = unique_count / max(non_missing, 1)
 
         if non_missing < min_non_missing:
-            dropped.append({"column": col, "reason": "insufficient_non_missing"})
+            drop(column, "insufficient_non_missing")
             continue
         if unique_count <= 1:
-            dropped.append({"column": col, "reason": "constant"})
+            drop(column, "constant")
             continue
-        if any(kw in lower for kw in _ID_KEYWORDS):
-            dropped.append({"column": col, "reason": "id_like_keyword"})
+        if _looks_like_identifier_name(column):
+            drop(column, "id_like_name")
             continue
-        if any(kw in lower for kw in _TIME_KEYWORDS) and not pd.api.types.is_numeric_dtype(
-            X[col]
-        ):
-            dropped.append({"column": col, "reason": "time_like_text"})
-            continue
-        if unique_ratio > drop_high_unique_ratio and not pd.api.types.is_numeric_dtype(X[col]):
-            dropped.append({"column": col, "reason": "high_unique_non_numeric"})
+        if _looks_like_time_name(column) and not pd.api.types.is_numeric_dtype(series):
+            drop(column, "time_like_non_numeric")
             continue
 
-    drop_cols = [d["column"] for d in dropped]
-    X = X.drop(columns=drop_cols, errors="ignore")
+        if _is_categorical_dtype(series):
+            if unique_count > max_categorical_levels:
+                drop(column, "high_cardinality_categorical")
+                continue
+            if unique_ratio > drop_high_unique_ratio:
+                drop(column, "high_unique_non_numeric")
+                continue
+
+        if (
+            not pd.api.types.is_numeric_dtype(series)
+            and not _is_categorical_dtype(series)
+        ):
+            drop(column, "unsupported_dtype")
+
+    X = X.drop(columns=list(dropped_names), errors="ignore")
 
     numeric_features = X.select_dtypes(include=[np.number]).columns.tolist()
-    categorical_features = X.select_dtypes(include=_CATEGORICAL_DTYPES).columns.tolist()
+    categorical_features = [
+        column
+        for column in X.columns
+        if _is_categorical_dtype(X[column])
+    ]
+
+    used_features = set(numeric_features) | set(categorical_features)
+    unused_columns = [column for column in X.columns if column not in used_features]
+    for column in unused_columns:
+        drop(column, "unsupported_dtype")
+    if unused_columns:
+        X = X.drop(columns=unused_columns, errors="ignore")
 
     total_features = len(numeric_features) + len(categorical_features)
     usable = total_features > 0 and len(y) >= 20
@@ -99,6 +217,9 @@ def prepare_ml_matrix(df, target, drop_high_unique_ratio=0.95, min_non_missing=5
         "categorical_features": categorical_features,
         "dropped_columns": dropped,
         "warnings": warnings,
+        "infinite_values_replaced": infinite_values_replaced,
+        "target_infinite_values_dropped": target_infinite_count,
+        "max_categorical_levels": int(max_categorical_levels),
         "usable": usable,
     }
 

From dc6298805121e10cc4b585769973dfbc981f5f63 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:10:15 +0530
Subject: [PATCH 082/439] test: cover ML preprocessing safety and
 false-positive fixes

---
 tests/test_ml_preprocessing_v2.py | 103 ++++++++++++++++++++++++++++++
 1 file changed, 103 insertions(+)
 create mode 100644 tests/test_ml_preprocessing_v2.py

diff --git a/tests/test_ml_preprocessing_v2.py b/tests/test_ml_preprocessing_v2.py
new file mode 100644
index 0000000..edbc4f5
--- /dev/null
+++ b/tests/test_ml_preprocessing_v2.py
@@ -0,0 +1,103 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from framevitals.ml_preprocessing import prepare_ml_matrix
+
+
+def test_id_detection_uses_boundaries_not_raw_substrings():
+    rows = 30
+    df = pd.DataFrame({
+        "paid_amount": [100 + index for index in range(rows)],
+        "width": [10 + (index % 5) for index in range(rows)],
+        "customer_id": [f"CUST-{index:03d}" for index in range(rows)],
+        "candidate_score": [50 + (index % 10) for index in range(rows)],
+        "target": [0, 1] * 15,
+    })
+
+    result = prepare_ml_matrix(df, target="target")
+    dropped = {item["column"]: item["reason"] for item in result["dropped_columns"]}
+
+    assert "paid_amount" in result["numeric_features"]
+    assert "width" in result["numeric_features"]
+    assert "candidate_score" in result["numeric_features"]
+    assert dropped["customer_id"] == "id_like_name"
+
+
+def test_numeric_infinities_are_replaced_for_imputation():
+    rows = 30
+    df = pd.DataFrame({
+        "measurement": [float(index) for index in range(rows)],
+        "target": [0, 1] * 15,
+    })
+    df.loc[3, "measurement"] = np.inf
+    df.loc[7, "measurement"] = -np.inf
+
+    result = prepare_ml_matrix(df, target="target")
+
+    assert result["usable"] is True
+    assert result["infinite_values_replaced"] == {"measurement": 2}
+    assert int(result["X"]["measurement"].isna().sum()) == 2
+    assert any("infinite values" in warning for warning in result["warnings"])
+
+
+def test_infinite_numeric_targets_are_dropped_as_invalid_labels():
+    rows = 30
+    target = [float(index) for index in range(rows)]
+    target[-1] = np.inf
+    df = pd.DataFrame({
+        "feature": list(range(rows)),
+        "target": target,
+    })
+
+    result = prepare_ml_matrix(df, target="target")
+
+    assert result["target_infinite_values_dropped"] == 1
+    assert len(result["y"]) == 29
+    assert np.isfinite(result["y"].to_numpy(dtype=float)).all()
+
+
+def test_high_cardinality_categorical_is_bounded_before_one_hot_encoding():
+    rows = 300
+    df = pd.DataFrame({
+        "category": [f"group-{index % 250}" for index in range(rows)],
+        "feature": [index % 12 for index in range(rows)],
+        "target": [0, 1] * 150,
+    })
+
+    result = prepare_ml_matrix(
+        df,
+        target="target",
+        max_categorical_levels=100,
+    )
+    dropped = {item["column"]: item["reason"] for item in result["dropped_columns"]}
+
+    assert dropped["category"] == "high_cardinality_categorical"
+    assert "category" not in result["categorical_features"]
+    assert "feature" in result["numeric_features"]
+
+
+def test_real_time_columns_are_excluded_but_unrelated_names_are_kept():
+    rows = 30
+    df = pd.DataFrame({
+        "event_date": pd.date_range("2026-01-01", periods=rows).astype(str),
+        "candidate_label": ["A", "B", "C"] * 10,
+        "target": [0, 1] * 15,
+    })
+
+    result = prepare_ml_matrix(df, target="target")
+    dropped = {item["column"]: item["reason"] for item in result["dropped_columns"]}
+
+    assert dropped["event_date"] == "time_like_non_numeric"
+    assert "candidate_label" in result["categorical_features"]
+
+
+def test_preprocessing_control_validation():
+    df = pd.DataFrame({"x": list(range(20)), "target": [0, 1] * 10})
+
+    with pytest.raises(ValueError, match="drop_high_unique_ratio"):
+        prepare_ml_matrix(df, "target", drop_high_unique_ratio=0)
+    with pytest.raises(ValueError, match="min_non_missing"):
+        prepare_ml_matrix(df, "target", min_non_missing=0)
+    with pytest.raises(ValueError, match="max_categorical_levels"):
+        prepare_ml_matrix(df, "target", max_categorical_levels=1)

From fb89a128d71b0a0b52a701fabe9af98f389a5df4 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:11:14 +0530
Subject: [PATCH 083/439] fix: harden model leaderboard validation and baseline
 interpretation

---
 src/framevitals/model_leaderboard.py | 462 ++++++++++++++++++---------
 1 file changed, 311 insertions(+), 151 deletions(-)

diff --git a/src/framevitals/model_leaderboard.py b/src/framevitals/model_leaderboard.py
index 84cb9be..e251af8 100644
--- a/src/framevitals/model_leaderboard.py
+++ b/src/framevitals/model_leaderboard.py
@@ -1,30 +1,9 @@
-"""
-Model Leaderboard (WS-3)
-========================
-Cross-validated leaderboard of multiple ML models for a given target.
-
-For classification:
-    DummyClassifier, LogisticRegression, KNeighborsClassifier,
-    RandomForestClassifier, GradientBoostingClassifier,
-    XGBClassifier (optional), LGBMClassifier (optional)
-
-For regression:
-    DummyRegressor, Ridge, Lasso, KNeighborsRegressor,
-    RandomForestRegressor, GradientBoostingRegressor,
-    XGBRegressor (optional), LGBMRegressor (optional)
-
-Each model is wrapped in a Pipeline with the shared preprocessor from
-modules.ml_preprocessing.prepare_ml_matrix + build_sklearn_preprocessor.
+"""Cross-validated baseline model leaderboard for FrameVitals.
 
-Validation:
-    StratifiedKFold(5) for classification, KFold(5) for regression.
-    XGBoost / LightGBM are imported lazily and skipped if missing.
-
-The winner is also fit on the full training data and a calibration check
-(classification) or residual summary (regression) is computed on a hold-out.
-
-Public entry point:
-    run_model_leaderboard(df, target_column, task_type=None) -> dict
+The leaderboard is a diagnostic for dataset learnability, not an AutoML system.
+It compares several lightweight/classical models (plus optional XGBoost and
+LightGBM), keeps preprocessing inside each CV fold, records failures, and
+compares the best real model against a dummy baseline.
 """
 
 from __future__ import annotations
@@ -63,9 +42,6 @@
     prepare_ml_matrix,
 )
 
-# ---------------------------------------------------------------------------
-# Optional heavy models — imported lazily so missing libs don't break import
-# ---------------------------------------------------------------------------
 
 def _maybe_xgb_classifier():
     try:
@@ -134,31 +110,25 @@ def _maybe_lgbm_regressor():
         return None
 
 
-# ---------------------------------------------------------------------------
-# Task auto-detection
-# ---------------------------------------------------------------------------
-
 def _infer_task_type(y: pd.Series) -> str:
     """Best-effort: classification if dtype is non-numeric or low-cardinality."""
     if pd.api.types.is_numeric_dtype(y):
         n = len(y)
         unique = int(y.nunique(dropna=True))
-        # Heuristic: integer-like with few unique values is classification
         if unique <= 20 and unique <= max(2, int(n * 0.05)):
             return "classification"
         return "regression"
     return "classification"
 
 
-# ---------------------------------------------------------------------------
-# Model registries
-# ---------------------------------------------------------------------------
-
 def _classification_registry(class_count: int) -> dict[str, Any]:
     registry: dict[str, Any] = {
         "DummyClassifier": DummyClassifier(strategy="most_frequent"),
         "LogisticRegression": LogisticRegression(
-            max_iter=2000, n_jobs=-1, class_weight="balanced", random_state=42
+            max_iter=2000,
+            n_jobs=-1,
+            class_weight="balanced",
+            random_state=42,
         ),
         "KNeighborsClassifier": KNeighborsClassifier(n_neighbors=7),
         "RandomForestClassifier": RandomForestClassifier(
@@ -169,7 +139,9 @@ def _classification_registry(class_count: int) -> dict[str, Any]:
             class_weight="balanced",
         ),
         "GradientBoostingClassifier": GradientBoostingClassifier(
-            n_estimators=150, max_depth=4, random_state=42
+            n_estimators=150,
+            max_depth=4,
+            random_state=42,
         ),
     }
     xgb = _maybe_xgb_classifier()
@@ -188,10 +160,15 @@ def _regression_registry() -> dict[str, Any]:
         "Lasso": Lasso(alpha=0.01, max_iter=20000, random_state=42),
         "KNeighborsRegressor": KNeighborsRegressor(n_neighbors=7),
         "RandomForestRegressor": RandomForestRegressor(
-            n_estimators=200, max_depth=10, random_state=42, n_jobs=-1
+            n_estimators=200,
+            max_depth=10,
+            random_state=42,
+            n_jobs=-1,
         ),
         "GradientBoostingRegressor": GradientBoostingRegressor(
-            n_estimators=150, max_depth=4, random_state=42
+            n_estimators=150,
+            max_depth=4,
+            random_state=42,
         ),
     }
     xgb = _maybe_xgb_regressor()
@@ -203,10 +180,6 @@ def _regression_registry() -> dict[str, Any]:
     return registry
 
 
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
 def _safe_round(value, ndigits: int = 4) -> float | None:
     if value is None:
         return None
@@ -220,55 +193,129 @@ def _safe_round(value, ndigits: int = 4) -> float | None:
 
 
 def _cv_for(task_type: str, y: pd.Series, n_splits: int = 5):
+    if n_splits < 2:
+        raise ValueError("n_splits must be at least 2.")
+
+    n_rows = len(y)
+    if n_rows < 2:
+        raise ValueError("At least 2 target rows are required for cross-validation.")
+
     if task_type == "classification":
-        # Reduce splits if smallest class is too tiny
-        min_class = int(y.value_counts().min()) if len(y) else 0
-        actual_splits = max(2, min(n_splits, min_class))
-        return StratifiedKFold(n_splits=actual_splits, shuffle=True, random_state=42), actual_splits
-    return KFold(n_splits=n_splits, shuffle=True, random_state=42), n_splits
+        counts = y.value_counts()
+        min_class = int(counts.min()) if len(counts) else 0
+        if min_class < 2:
+            raise ValueError(
+                "Each classification class needs at least 2 rows for stratified cross-validation."
+            )
+        actual_splits = min(n_splits, min_class)
+        return (
+            StratifiedKFold(
+                n_splits=actual_splits,
+                shuffle=True,
+                random_state=42,
+            ),
+            actual_splits,
+        )
+
+    actual_splits = min(n_splits, n_rows)
+    if actual_splits < 2:
+        raise ValueError("Regression cross-validation needs at least 2 folds.")
+    return (
+        KFold(n_splits=actual_splits, shuffle=True, random_state=42),
+        actual_splits,
+    )
 
 
 def _scoring_for(task_type: str) -> tuple[dict[str, str], str]:
     if task_type == "classification":
-        scoring = {
+        return {
             "accuracy": "accuracy",
             "f1_weighted": "f1_weighted",
             "precision_weighted": "precision_weighted",
             "recall_weighted": "recall_weighted",
-        }
-        return scoring, "f1_weighted"
-    scoring = {
+        }, "f1_weighted"
+    return {
         "r2": "r2",
         "neg_mae": "neg_mean_absolute_error",
         "neg_rmse": "neg_root_mean_squared_error",
-    }
-    return scoring, "r2"
+    }, "r2"
+
+
+def _encode_classification_target(
+    y: pd.Series,
+) -> tuple[pd.Series, list[dict[str, Any]]]:
+    """Encode every classification target to stable consecutive integer labels."""
+    codes, uniques = pd.factorize(y, sort=True)
+    encoded = pd.Series(codes, index=y.index, name=y.name, dtype="int64")
+    mapping = [
+        {
+            "encoded": int(index),
+            "label": value.item() if isinstance(value, np.generic) else str(value),
+        }
+        for index, value in enumerate(uniques.tolist())
+    ]
+    return encoded, mapping
 
 
-def _coerce_classification_target(y: pd.Series) -> pd.Series:
-    """XGBoost requires integer-encoded class labels; encode safely."""
-    if pd.api.types.is_numeric_dtype(y):
-        return y
-    return y.astype("category").cat.codes
+def _set_estimator_jobs(estimator: Any, jobs: int) -> Any:
+    """Avoid nested process/thread explosions inside parallel CV folds."""
+    try:
+        params = estimator.get_params(deep=False)
+        if "n_jobs" in params:
+            estimator.set_params(n_jobs=jobs)
+    except Exception:
+        pass
+    return estimator
+
+
+def _adapt_knn_neighbors(registry: dict[str, Any], n_rows: int, n_splits: int) -> int:
+    largest_test_fold = int(np.ceil(n_rows / n_splits))
+    smallest_train_fold = max(1, n_rows - largest_test_fold)
+    neighbors = max(1, min(7, smallest_train_fold))
+    for name in ("KNeighborsClassifier", "KNeighborsRegressor"):
+        estimator = registry.get(name)
+        if estimator is not None:
+            try:
+                estimator.set_params(n_neighbors=neighbors)
+            except Exception:
+                pass
+    return neighbors
+
 
+def _score_stability(std: float | None) -> str:
+    if std is None:
+        return "unknown"
+    if std <= 0.02:
+        return "stable"
+    if std <= 0.05:
+        return "moderate"
+    return "variable"
 
-# ---------------------------------------------------------------------------
-# Hold-out diagnostics for the winner
-# ---------------------------------------------------------------------------
 
 def _classification_holdout(
-    pipeline: Pipeline, X: pd.DataFrame, y: pd.Series
-) -> dict:
+    pipeline: Pipeline,
+    X: pd.DataFrame,
+    y: pd.Series,
+) -> dict[str, Any]:
     n = len(y)
     test_size = max(50, int(n * 0.25)) if n >= 200 else max(20, int(n * 0.25))
     test_size = min(test_size, n - 20)
     if test_size <= 0:
         return {"available": False, "reason": "not enough rows for holdout"}
 
-    stratify = y if y.nunique() > 1 and y.value_counts().min() >= 2 else None
+    class_count = int(y.nunique())
+    stratify = y if class_count > 1 and y.value_counts().min() >= 2 else None
+    if stratify is not None:
+        test_size = max(test_size, class_count)
+        if n - test_size < class_count:
+            stratify = None
 
     X_train, X_test, y_train, y_test = train_test_split(
-        X, y, test_size=test_size, random_state=42, stratify=stratify
+        X,
+        y,
+        test_size=test_size,
+        random_state=42,
+        stratify=stratify,
     )
 
     pipeline.fit(X_train, y_train)
@@ -279,7 +326,9 @@ def _classification_holdout(
         "n_train": int(len(X_train)),
         "n_test": int(len(X_test)),
         "accuracy": _safe_round(accuracy_score(y_test, y_pred)),
-        "f1_weighted": _safe_round(f1_score(y_test, y_pred, average="weighted", zero_division=0)),
+        "f1_weighted": _safe_round(
+            f1_score(y_test, y_pred, average="weighted", zero_division=0)
+        ),
         "precision_weighted": _safe_round(
             precision_score(y_test, y_pred, average="weighted", zero_division=0)
         ),
@@ -288,23 +337,27 @@ def _classification_holdout(
         ),
     }
 
-    # Brier score for binary calibration
     try:
         if hasattr(pipeline, "predict_proba"):
             proba = pipeline.predict_proba(X_test)
             classes = pipeline.classes_ if hasattr(pipeline, "classes_") else None
             if proba.shape[1] == 2 and classes is not None:
-                # Binary: pick positive-class column
                 pos_idx = 1
                 y_bin = (y_test == classes[pos_idx]).astype(int).values
-                out["brier_score"] = _safe_round(brier_score_loss(y_bin, proba[:, pos_idx]))
+                out["brier_score"] = _safe_round(
+                    brier_score_loss(y_bin, proba[:, pos_idx])
+                )
     except Exception as exc:
         out["brier_error"] = str(exc)
 
     return out
 
 
-def _regression_holdout(pipeline: Pipeline, X: pd.DataFrame, y: pd.Series) -> dict:
+def _regression_holdout(
+    pipeline: Pipeline,
+    X: pd.DataFrame,
+    y: pd.Series,
+) -> dict[str, Any]:
     n = len(y)
     test_size = max(50, int(n * 0.25)) if n >= 200 else max(20, int(n * 0.25))
     test_size = min(test_size, n - 20)
@@ -312,11 +365,13 @@ def _regression_holdout(pipeline: Pipeline, X: pd.DataFrame, y: pd.Series) -> di
         return {"available": False, "reason": "not enough rows for holdout"}
 
     X_train, X_test, y_train, y_test = train_test_split(
-        X, y, test_size=test_size, random_state=42
+        X,
+        y,
+        test_size=test_size,
+        random_state=42,
     )
     pipeline.fit(X_train, y_train)
     y_pred = pipeline.predict(X_test)
-
     residuals = y_test.values - y_pred
 
     return {
@@ -335,30 +390,25 @@ def _regression_holdout(pipeline: Pipeline, X: pd.DataFrame, y: pd.Series) -> di
     }
 
 
-# ---------------------------------------------------------------------------
-# Public entry point
-# ---------------------------------------------------------------------------
-
 def run_model_leaderboard(
     df: pd.DataFrame,
     target_column: str,
     task_type: str | None = None,
     n_splits: int = 5,
-) -> dict:
-    """
-    Run a CV-validated model leaderboard.
-
-    Args:
-        df: Dataframe.
-        target_column: Target column name.
-        task_type: "classification" | "regression" | None (auto-detect).
-        n_splits: CV folds (capped by smallest class for classification).
-
-    Returns:
-        JSON-safe dict with leaderboard rows, winner card, and holdout metrics.
-    """
+    n_jobs: int = 1,
+) -> dict[str, Any]:
+    """Run a CV-validated diagnostic model leaderboard."""
     if not target_column or target_column not in df.columns:
-        return {"available": False, "message": f"Target '{target_column}' not in dataframe."}
+        return {
+            "available": False,
+            "message": f"Target '{target_column}' not in dataframe.",
+        }
+    if task_type is not None and task_type not in {"classification", "regression"}:
+        raise ValueError("task_type must be 'classification', 'regression', or None.")
+    if n_splits < 2:
+        raise ValueError("n_splits must be at least 2.")
+    if n_jobs == 0:
+        raise ValueError("n_jobs cannot be 0.")
 
     prep = prepare_ml_matrix(df, target=target_column)
     if not prep["usable"]:
@@ -366,6 +416,7 @@ def run_model_leaderboard(
             "available": False,
             "message": "; ".join(prep["warnings"]) or "Insufficient data for ML.",
             "dropped_columns": prep["dropped_columns"],
+            "warnings": prep["warnings"],
         }
 
     X = prep["X"]
@@ -376,28 +427,68 @@ def run_model_leaderboard(
     if task_type is None:
         task_type = _infer_task_type(y)
 
-    # XGBoost requires integer-encoded labels for classification
-    y_for_models = _coerce_classification_target(y) if task_type == "classification" else y
+    target_encoding: list[dict[str, Any]] | None = None
+    if task_type == "classification":
+        y_for_models, target_encoding = _encode_classification_target(y)
+        if y_for_models.nunique() < 2:
+            return {"available": False, "message": "Target has <2 classes."}
+    else:
+        if not pd.api.types.is_numeric_dtype(y):
+            return {
+                "available": False,
+                "message": "Regression requires a numeric target.",
+            }
+        y_for_models = pd.to_numeric(y, errors="coerce")
+        if not np.isfinite(y_for_models.to_numpy(dtype=float)).all():
+            return {
+                "available": False,
+                "message": "Regression target contains non-finite values after preprocessing.",
+            }
 
-    if task_type == "classification" and y_for_models.nunique() < 2:
-        return {"available": False, "message": "Target has <2 classes."}
+    try:
+        cv, actual_splits = _cv_for(
+            task_type,
+            y_for_models,
+            n_splits=n_splits,
+        )
+    except ValueError as exc:
+        return {
+            "available": False,
+            "task_type": task_type,
+            "target_column": target_column,
+            "message": str(exc),
+        }
 
-    cv, actual_splits = _cv_for(task_type, y_for_models, n_splits=n_splits)
     scoring, primary = _scoring_for(task_type)
-
     if task_type == "classification":
-        registry = _classification_registry(class_count=int(y_for_models.nunique()))
+        registry = _classification_registry(
+            class_count=int(y_for_models.nunique())
+        )
     else:
         registry = _regression_registry()
 
-    leaderboard: list[dict] = []
+    knn_neighbors = _adapt_knn_neighbors(
+        registry,
+        n_rows=len(y_for_models),
+        n_splits=actual_splits,
+    )
+    for estimator in registry.values():
+        _set_estimator_jobs(estimator, 1)
+
+    leaderboard: list[dict[str, Any]] = []
 
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
 
-        for name, est in registry.items():
-            preprocessor = build_sklearn_preprocessor(numeric_features, categorical_features)
-            pipeline = Pipeline([("pre", preprocessor), ("model", est)])
+        for name, estimator in registry.items():
+            preprocessor = build_sklearn_preprocessor(
+                numeric_features,
+                categorical_features,
+            )
+            pipeline = Pipeline([
+                ("pre", preprocessor),
+                ("model", estimator),
+            ])
 
             t0 = time.perf_counter()
             try:
@@ -407,95 +498,164 @@ def run_model_leaderboard(
                     y_for_models,
                     cv=cv,
                     scoring=scoring,
-                    n_jobs=-1,
+                    n_jobs=n_jobs,
                     return_train_score=False,
                     error_score=np.nan,
                 )
-                fit_time_s = float(np.mean(cv_results["fit_time"]))
                 row: dict[str, Any] = {
                     "model": name,
-                    "fit_time_s": _safe_round(fit_time_s, ndigits=3),
-                    "cv_total_s": _safe_round(time.perf_counter() - t0, ndigits=3),
+                    "fit_time_s": _safe_round(
+                        np.nanmean(cv_results["fit_time"]),
+                        ndigits=3,
+                    ),
+                    "cv_total_s": _safe_round(
+                        time.perf_counter() - t0,
+                        ndigits=3,
+                    ),
                     "n_splits": actual_splits,
                 }
-                for metric_key in scoring.keys():
-                    arr = cv_results[f"test_{metric_key}"]
+                for metric_key in scoring:
+                    arr = np.asarray(cv_results[f"test_{metric_key}"], dtype=float)
+                    valid_folds = int(np.isfinite(arr).sum())
                     if metric_key.startswith("neg_"):
                         clean_key = metric_key[4:]
                         row[f"{clean_key}_mean"] = _safe_round(-np.nanmean(arr))
                         row[f"{clean_key}_std"] = _safe_round(np.nanstd(arr))
+                        row[f"{clean_key}_valid_folds"] = valid_folds
                     else:
                         row[f"{metric_key}_mean"] = _safe_round(np.nanmean(arr))
                         row[f"{metric_key}_std"] = _safe_round(np.nanstd(arr))
-                row["primary_score"] = (
-                    row.get(f"{primary}_mean")
-                    if not primary.startswith("neg_")
-                    else row.get(f"{primary[4:]}_mean")
-                )
+                        row[f"{metric_key}_valid_folds"] = valid_folds
+
+                row["primary_score"] = row.get(f"{primary}_mean")
+                row["primary_std"] = row.get(f"{primary}_std")
+                row["score_stability"] = _score_stability(row["primary_std"])
+                if row["primary_score"] is None:
+                    row["error"] = "No valid cross-validation score was produced."
                 leaderboard.append(row)
             except Exception as exc:
-                leaderboard.append({"model": name, "error": str(exc)})
+                leaderboard.append({
+                    "model": name,
+                    "error": f"{type(exc).__name__}: {exc}",
+                })
 
-    # Sort by primary metric (higher is better for accuracy/f1/r2; for MAE/RMSE we use neg_)
-    def _sort_key(row: dict) -> float:
-        val = row.get("primary_score")
-        return -float("inf") if val is None else float(val)
+    def _sort_key(row: dict[str, Any]) -> float:
+        value = row.get("primary_score")
+        return -float("inf") if value is None else float(value)
 
     leaderboard.sort(key=_sort_key, reverse=True)
 
-    # Pick best non-dummy model
+    successful_rows = [
+        row for row in leaderboard if row.get("primary_score") is not None
+    ]
+    failed_rows = [row for row in leaderboard if row.get("primary_score") is None]
+    dummy_row = next(
+        (row for row in successful_rows if "Dummy" in row["model"]),
+        None,
+    )
     candidates = [
-        r for r in leaderboard
-        if r.get("primary_score") is not None and "Dummy" not in r["model"]
+        row for row in successful_rows if "Dummy" not in row["model"]
     ]
+
+    base_payload: dict[str, Any] = {
+        "available": True,
+        "task_type": task_type,
+        "target_column": target_column,
+        "primary_metric": primary,
+        "n_rows": int(len(y_for_models)),
+        "n_features": len(numeric_features) + len(categorical_features),
+        "numeric_features": numeric_features,
+        "categorical_features": categorical_features,
+        "dropped_columns": prep["dropped_columns"],
+        "warnings": list(prep["warnings"]),
+        "target_encoding": target_encoding,
+        "cv": {
+            "requested_splits": int(n_splits),
+            "actual_splits": int(actual_splits),
+            "n_jobs": int(n_jobs),
+            "knn_neighbors": int(knn_neighbors),
+        },
+        "models_succeeded": len(successful_rows),
+        "models_failed": len(failed_rows),
+        "model_failures": [
+            {"model": row["model"], "error": row.get("error")}
+            for row in failed_rows
+        ],
+        "leaderboard": leaderboard,
+        "baseline": (
+            {
+                "model": dummy_row["model"],
+                "primary_score": dummy_row["primary_score"],
+            }
+            if dummy_row is not None
+            else None
+        ),
+    }
+
     if not candidates:
         return {
-            "available": True,
-            "task_type": task_type,
-            "target_column": target_column,
-            "n_rows": int(len(y)),
-            "n_features": len(numeric_features) + len(categorical_features),
-            "numeric_features": numeric_features,
-            "categorical_features": categorical_features,
-            "leaderboard": leaderboard,
+            **base_payload,
             "winner": None,
             "message": "No non-dummy model produced a usable score.",
         }
 
     winner = candidates[0]
+    baseline_score = (
+        float(dummy_row["primary_score"])
+        if dummy_row is not None and dummy_row.get("primary_score") is not None
+        else None
+    )
+    winner_score = float(winner["primary_score"])
+    lift = winner_score - baseline_score if baseline_score is not None else None
+    beats_baseline = bool(lift is not None and lift > 0)
+
+    if baseline_score is not None and not beats_baseline:
+        base_payload["warnings"].append(
+            "Best non-dummy model did not outperform the dummy baseline on the primary CV metric."
+        )
 
-    # Refit winner on full data and run holdout
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
-
         winner_estimator = registry[winner["model"]]
-        winner_preprocessor = build_sklearn_preprocessor(numeric_features, categorical_features)
-        winner_pipeline = Pipeline([("pre", winner_preprocessor), ("model", winner_estimator)])
+        winner_preprocessor = build_sklearn_preprocessor(
+            numeric_features,
+            categorical_features,
+        )
+        winner_pipeline = Pipeline([
+            ("pre", winner_preprocessor),
+            ("model", winner_estimator),
+        ])
 
         try:
             if task_type == "classification":
-                holdout = _classification_holdout(winner_pipeline, X, y_for_models)
+                holdout = _classification_holdout(
+                    winner_pipeline,
+                    X,
+                    y_for_models,
+                )
             else:
-                holdout = _regression_holdout(winner_pipeline, X, y_for_models)
+                holdout = _regression_holdout(
+                    winner_pipeline,
+                    X,
+                    y_for_models,
+                )
         except Exception as exc:
-            holdout = {"available": False, "error": str(exc)}
+            holdout = {
+                "available": False,
+                "error": f"{type(exc).__name__}: {exc}",
+            }
 
     return {
-        "available": True,
-        "task_type": task_type,
-        "target_column": target_column,
-        "primary_metric": primary,
-        "n_rows": int(len(y)),
-        "n_features": len(numeric_features) + len(categorical_features),
-        "numeric_features": numeric_features,
-        "categorical_features": categorical_features,
-        "dropped_columns": prep["dropped_columns"],
-        "warnings": prep["warnings"],
-        "leaderboard": leaderboard,
+        **base_payload,
         "winner": {
             "model": winner["model"],
             "primary_score": winner["primary_score"],
-            "fit_time_s": winner["fit_time_s"],
+            "primary_std": winner.get("primary_std"),
+            "score_stability": winner.get("score_stability"),
+            "fit_time_s": winner.get("fit_time_s"),
+            "baseline_score": _safe_round(baseline_score),
+            "lift_over_baseline": _safe_round(lift),
+            "beats_baseline": beats_baseline if baseline_score is not None else None,
             "holdout": holdout,
         },
     }

From 39c5feabae3fd67dd62c4509f21a9362a2d77a01 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:11:39 +0530
Subject: [PATCH 084/439] test: cover robust leaderboard CV encoding and
 baselines

---
 tests/test_model_leaderboard_v2.py | 141 +++++++++++++++++++++++++++++
 1 file changed, 141 insertions(+)
 create mode 100644 tests/test_model_leaderboard_v2.py

diff --git a/tests/test_model_leaderboard_v2.py b/tests/test_model_leaderboard_v2.py
new file mode 100644
index 0000000..f680683
--- /dev/null
+++ b/tests/test_model_leaderboard_v2.py
@@ -0,0 +1,141 @@
+import pandas as pd
+import pytest
+from sklearn.dummy import DummyClassifier, DummyRegressor
+from sklearn.linear_model import LogisticRegression, Ridge
+
+import framevitals.model_leaderboard as leaderboard_module
+
+
+def _classification_frame(labels=(10, 20), rows=40):
+    return pd.DataFrame({
+        "feature": list(range(rows)),
+        "segment": ["A", "B"] * (rows // 2),
+        "target": [labels[index % len(labels)] for index in range(rows)],
+    })
+
+
+def test_non_consecutive_numeric_class_labels_are_encoded_safely(monkeypatch):
+    def registry(class_count):
+        return {
+            "DummyClassifier": DummyClassifier(strategy="most_frequent"),
+            "LogisticRegression": LogisticRegression(max_iter=500, random_state=42),
+        }
+
+    monkeypatch.setattr(leaderboard_module, "_classification_registry", registry)
+    result = leaderboard_module.run_model_leaderboard(
+        _classification_frame(labels=(10, 20)),
+        target_column="target",
+        task_type="classification",
+        n_splits=2,
+    )
+
+    assert result["available"] is True
+    assert result["target_encoding"] == [
+        {"encoded": 0, "label": 10},
+        {"encoded": 1, "label": 20},
+    ]
+    assert result["winner"] is not None
+    assert result["models_succeeded"] >= 2
+    assert result["baseline"]["model"] == "DummyClassifier"
+
+
+def test_rare_class_returns_clear_cv_unavailable_result(monkeypatch):
+    df = pd.DataFrame({
+        "feature": list(range(25)),
+        "target": [0] * 24 + [1],
+    })
+
+    def registry(class_count):
+        return {"LogisticRegression": LogisticRegression(max_iter=200)}
+
+    monkeypatch.setattr(leaderboard_module, "_classification_registry", registry)
+    result = leaderboard_module.run_model_leaderboard(
+        df,
+        target_column="target",
+        task_type="classification",
+        n_splits=5,
+    )
+
+    assert result["available"] is False
+    assert "at least 2 rows" in result["message"]
+
+
+def test_regression_cv_caps_requested_folds_and_reports_baseline(monkeypatch):
+    rows = 24
+    df = pd.DataFrame({
+        "feature": list(range(rows)),
+        "target": [float(index * 3 + 1) for index in range(rows)],
+    })
+
+    def registry():
+        return {
+            "DummyRegressor": DummyRegressor(strategy="mean"),
+            "Ridge": Ridge(alpha=1.0),
+        }
+
+    monkeypatch.setattr(leaderboard_module, "_regression_registry", registry)
+    result = leaderboard_module.run_model_leaderboard(
+        df,
+        target_column="target",
+        task_type="regression",
+        n_splits=100,
+    )
+
+    assert result["available"] is True
+    assert result["cv"]["requested_splits"] == 100
+    assert result["cv"]["actual_splits"] == rows
+    assert result["baseline"]["model"] == "DummyRegressor"
+    assert result["winner"]["model"] == "Ridge"
+    assert result["winner"]["beats_baseline"] is True
+    assert result["winner"]["lift_over_baseline"] is not None
+
+
+def test_leaderboard_records_model_failures_without_sinking_other_models(monkeypatch):
+    class BrokenEstimator:
+        def get_params(self, deep=True):
+            return {}
+
+        def fit(self, X, y):
+            raise RuntimeError("broken intentionally")
+
+    def registry(class_count):
+        return {
+            "Broken": BrokenEstimator(),
+            "LogisticRegression": LogisticRegression(max_iter=500),
+        }
+
+    monkeypatch.setattr(leaderboard_module, "_classification_registry", registry)
+    result = leaderboard_module.run_model_leaderboard(
+        _classification_frame(labels=(0, 1)),
+        target_column="target",
+        task_type="classification",
+        n_splits=2,
+    )
+
+    assert result["available"] is True
+    assert result["winner"]["model"] == "LogisticRegression"
+    assert result["models_failed"] == 1
+    assert result["model_failures"][0]["model"] == "Broken"
+
+
+def test_leaderboard_validates_controls():
+    df = _classification_frame(labels=(0, 1))
+
+    with pytest.raises(ValueError, match="task_type"):
+        leaderboard_module.run_model_leaderboard(
+            df,
+            target_column="target",
+            task_type="ranking",
+        )
+    with pytest.raises(ValueError, match="n_splits"):
+        leaderboard_module.run_model_leaderboard(
+            df,
+            target_column="target",
+            n_splits=1,
+        )
+    with pytest.raises(ValueError, match="n_jobs"):
+        leaderboard_module.run_model_leaderboard(
+            df,
+            target_column="target",
+            n_jobs=0,
+        )

From 6db65f68113c342da0889d728d74ff9f2263a4df Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:15:45 +0530
Subject: [PATCH 085/439] fix: preserve target labels and bound regression CV
 folds

---
 src/framevitals/model_leaderboard.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/src/framevitals/model_leaderboard.py b/src/framevitals/model_leaderboard.py
index e251af8..7bbfc53 100644
--- a/src/framevitals/model_leaderboard.py
+++ b/src/framevitals/model_leaderboard.py
@@ -217,7 +217,12 @@ def _cv_for(task_type: str, y: pd.Series, n_splits: int = 5):
             actual_splits,
         )
 
-    actual_splits = min(n_splits, n_rows)
+    # R² is undefined for a test fold containing fewer than two observations.
+    # Because ML preprocessing already requires >=20 rows, capping KFold at
+    # floor(n/2) guarantees every test fold has at least two rows while still
+    # honouring smaller user-requested fold counts.
+    max_r2_splits = max(2, n_rows // 2)
+    actual_splits = min(n_splits, max_r2_splits)
     if actual_splits < 2:
         raise ValueError("Regression cross-validation needs at least 2 folds.")
     return (
@@ -241,16 +246,29 @@ def _scoring_for(task_type: str) -> tuple[dict[str, str], str]:
     }, "r2"
 
 
+def _json_safe_label(value: Any) -> Any:
+    """Preserve ordinary class-label types while keeping metadata JSON-safe."""
+    if isinstance(value, np.generic):
+        value = value.item()
+    if isinstance(value, pd.Timestamp):
+        return value.isoformat()
+    if isinstance(value, (str, int, float, bool)) or value is None:
+        return value
+    return str(value)
+
+
 def _encode_classification_target(
     y: pd.Series,
 ) -> tuple[pd.Series, list[dict[str, Any]]]:
     """Encode every classification target to stable consecutive integer labels."""
-    codes, uniques = pd.factorize(y, sort=True)
+    # sort=False also supports heterogeneous object labels that cannot be
+    # ordered against one another (for example a mix of strings and integers).
+    codes, uniques = pd.factorize(y, sort=False)
     encoded = pd.Series(codes, index=y.index, name=y.name, dtype="int64")
     mapping = [
         {
             "encoded": int(index),
-            "label": value.item() if isinstance(value, np.generic) else str(value),
+            "label": _json_safe_label(value),
         }
         for index, value in enumerate(uniques.tolist())
     ]

From 5c4f8288974cce2101978c2893afa44167ef3cc9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:16:08 +0530
Subject: [PATCH 086/439] test: require regression CV folds with valid R2 test
 sizes

---
 tests/test_model_leaderboard_v2.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_model_leaderboard_v2.py b/tests/test_model_leaderboard_v2.py
index f680683..5fe1ca3 100644
--- a/tests/test_model_leaderboard_v2.py
+++ b/tests/test_model_leaderboard_v2.py
@@ -83,7 +83,9 @@ def registry():
 
     assert result["available"] is True
     assert result["cv"]["requested_splits"] == 100
-    assert result["cv"]["actual_splits"] == rows
+    # R2 needs at least two observations in each test fold, so the safe upper
+    # bound is floor(n_rows / 2), not one fold per row.
+    assert result["cv"]["actual_splits"] == rows // 2
     assert result["baseline"]["model"] == "DummyRegressor"
     assert result["winner"]["model"] == "Ridge"
     assert result["winner"]["beats_baseline"] is True

From 59d643129954dab0438e8654cc33e15ebc9e6301 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:16:58 +0530
Subject: [PATCH 087/439] feat: add focused public analysis APIs

---
 src/framevitals/api.py | 129 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 122 insertions(+), 7 deletions(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index bd37ae9..3aae427 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -8,23 +8,29 @@
 import pandas as pd
 
 from framevitals.analysis_selector import select_analyses
+from framevitals.anomaly_ensemble import detect_anomalies_ensemble
 from framevitals.cleaning_plan import (
     CleaningPlan,
     apply_cleaning_plan,
     infer_cleaning_plan,
 )
-from framevitals.column_roles import infer_column_roles
+from framevitals.column_roles import infer_column_roles, summarize_roles
 from framevitals.config import ConfigInput, VALID_MODULES, resolve_config
 from framevitals.contracts import infer_contract as _infer_contract
 from framevitals.contracts import validate_contract
 from framevitals.dataset_signals import detect_dataset_signals
+from framevitals.deep_statistics_v2 import run_deep_statistics_v2
 from framevitals.drift_analysis import compare_datasets
+from framevitals.health_score import calculate_health_score
 from framevitals.loader import load_dataset
+from framevitals.ml_readiness import calculate_ml_readiness
 from framevitals.pipeline import run_full_analysis
 from framevitals.planning import AnalysisPlan
 from framevitals.profiler import build_profile
+from framevitals.quality_diagnostics import run_quality_diagnostics
 from framevitals.quality_results import DriftResult, ValidationResult
 from framevitals.result import AnalysisResult
+from framevitals.target_intelligence import run_target_intelligence
 
 DataInput = str | Path | pd.DataFrame
 
@@ -56,6 +62,115 @@ def _load_input(value: DataInput, *, label: str) -> tuple[pd.DataFrame, str]:
     )
 
 
+def _with_dataset_name(payload: dict[str, Any], source_name: str) -> dict[str, Any]:
+    """Attach source identity without mutating an analysis module's own result."""
+    return {"dataset_name": source_name, **payload}
+
+
+def profile(data: DataInput) -> dict[str, Any]:
+    """Profile shape, types, missingness, cardinality, and basic summaries only."""
+    dataframe, source_name = _load_input(data, label="Dataset")
+    return _with_dataset_name(build_profile(dataframe), source_name)
+
+
+def roles(data: DataInput) -> dict[str, Any]:
+    """Infer semantic/structural column roles without running the full pipeline."""
+    dataframe, source_name = _load_input(data, label="Dataset")
+    column_roles = infer_column_roles(dataframe)
+    return {
+        "dataset_name": source_name,
+        "columns": column_roles,
+        "summary": summarize_roles(column_roles),
+    }
+
+
+def health(data: DataInput) -> dict[str, Any]:
+    """Calculate the FrameVitals data-health score only."""
+    dataframe, source_name = _load_input(data, label="Dataset")
+    dataset_profile = build_profile(dataframe)
+    payload = calculate_health_score(dataframe, dataset_profile)
+    return _with_dataset_name(payload, source_name)
+
+
+def ml_readiness(data: DataInput) -> dict[str, Any]:
+    """Calculate ML-readiness diagnostics only."""
+    dataframe, source_name = _load_input(data, label="Dataset")
+    dataset_profile = build_profile(dataframe)
+    payload = calculate_ml_readiness(dataframe, profile=dataset_profile)
+    return _with_dataset_name(payload, source_name)
+
+
+def quality(
+    data: DataInput,
+    *,
+    max_sample_rows: int = 5_000,
+    max_columns: int = 100,
+    max_missingness_columns: int = 25,
+) -> dict[str, Any]:
+    """Run practical deterministic data-quality diagnostics only."""
+    dataframe, source_name = _load_input(data, label="Dataset")
+    dataset_profile = build_profile(dataframe)
+    column_roles = infer_column_roles(dataframe)
+    payload = run_quality_diagnostics(
+        dataframe,
+        profile=dataset_profile,
+        column_roles=column_roles,
+        max_sample_rows=max_sample_rows,
+        max_columns=max_columns,
+        max_missingness_columns=max_missingness_columns,
+    )
+    return _with_dataset_name(payload, source_name)
+
+
+def statistics(
+    data: DataInput,
+    *,
+    max_pairs: int = 20,
+) -> dict[str, Any]:
+    """Run the deep statistical diagnostics layer only."""
+    dataframe, source_name = _load_input(data, label="Dataset")
+    payload = run_deep_statistics_v2(dataframe, max_pairs=max_pairs)
+    return _with_dataset_name(payload, source_name)
+
+
+def anomalies(
+    data: DataInput,
+    *,
+    contamination: float = 0.05,
+    threshold: float = 0.6,
+    max_columns: int = 30,
+    top_k: int = 25,
+) -> dict[str, Any]:
+    """Run the tabular anomaly ensemble only."""
+    dataframe, source_name = _load_input(data, label="Dataset")
+    payload = detect_anomalies_ensemble(
+        dataframe,
+        contamination=contamination,
+        threshold=threshold,
+        max_columns=max_columns,
+        top_k=top_k,
+    )
+    return _with_dataset_name(payload, source_name)
+
+
+def target_analysis(
+    data: DataInput,
+    *,
+    target: str,
+) -> dict[str, Any]:
+    """Run target-quality, leakage, association, and split diagnostics only."""
+    dataframe, source_name = _load_input(data, label="Dataset")
+    if target not in dataframe.columns:
+        raise ValueError(f"Target column not found: {target}")
+    column_roles = infer_column_roles(dataframe)
+    payload = run_target_intelligence(
+        dataframe,
+        target_column=target,
+        column_roles=column_roles,
+    )
+    return _with_dataset_name(payload, source_name)
+
+
 def analyze(
     data: DataInput,
     *,
@@ -67,7 +182,7 @@ def analyze(
     config: ConfigInput = None,
     disabled_modules: list[str] | tuple[str, ...] | None = None,
 ) -> AnalysisResult:
-    """Analyze a tabular dataset with FrameVitals."""
+    """Analyze a tabular dataset with the complete configured FrameVitals pipeline."""
     resolved = resolve_config(
         config,
         preset=preset,
@@ -139,11 +254,11 @@ def plan(
         disabled_modules=disabled_modules,
     )
     dataframe, source_name = _load_input(data, label="Dataset")
-    profile = build_profile(dataframe)
+    dataset_profile = build_profile(dataframe)
     column_roles = infer_column_roles(dataframe)
     dataset_signals = detect_dataset_signals(
         dataframe,
-        profile,
+        dataset_profile,
         column_roles=column_roles,
     )
     selection = select_analyses(
@@ -166,7 +281,7 @@ def plan(
         "dataset_name": source_name,
         "analysis_mode": resolved.mode,
         "target": resolved.target,
-        "shape": dict(profile.get("shape", {})),
+        "shape": dict(dataset_profile.get("shape", {})),
         "config": resolved.to_dict(),
         "signals": public_signals,
         "selection": selection,
@@ -176,8 +291,8 @@ def plan(
 def plan_cleaning(data: DataInput) -> CleaningPlan:
     """Infer a conservative cleaning plan without modifying the input data."""
     dataframe, _ = _load_input(data, label="Dataset")
-    profile = build_profile(dataframe)
-    return infer_cleaning_plan(dataframe, profile=profile)
+    dataset_profile = build_profile(dataframe)
+    return infer_cleaning_plan(dataframe, profile=dataset_profile)
 
 
 def clean(

From 486e6611e1072dfbbf15fc48395550800bc9b4f6 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:17:28 +0530
Subject: [PATCH 088/439] feat: expose focused analysis functions lazily

---
 src/framevitals/__init__.py | 88 +++++++++++++++++++++++++++++++++++++
 1 file changed, 88 insertions(+)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 3d6f3dc..3cc89ed 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -35,6 +35,86 @@ def __getattr__(name: str):
     raise AttributeError(name)
 
 
+def profile(data: Any) -> dict[str, Any]:
+    """Profile shape, dtypes, missingness, cardinality, and basic summaries."""
+    from framevitals.api import profile as _profile
+
+    return _profile(data)
+
+
+def roles(data: Any) -> dict[str, Any]:
+    """Infer semantic and structural roles for dataset columns."""
+    from framevitals.api import roles as _roles
+
+    return _roles(data)
+
+
+def health(data: Any) -> dict[str, Any]:
+    """Calculate only the FrameVitals data-health score."""
+    from framevitals.api import health as _health
+
+    return _health(data)
+
+
+def ml_readiness(data: Any) -> dict[str, Any]:
+    """Calculate only ML-readiness diagnostics."""
+    from framevitals.api import ml_readiness as _ml_readiness
+
+    return _ml_readiness(data)
+
+
+def quality(
+    data: Any,
+    *,
+    max_sample_rows: int = 5_000,
+    max_columns: int = 100,
+    max_missingness_columns: int = 25,
+) -> dict[str, Any]:
+    """Run practical deterministic data-quality diagnostics only."""
+    from framevitals.api import quality as _quality
+
+    return _quality(
+        data,
+        max_sample_rows=max_sample_rows,
+        max_columns=max_columns,
+        max_missingness_columns=max_missingness_columns,
+    )
+
+
+def statistics(data: Any, *, max_pairs: int = 20) -> dict[str, Any]:
+    """Run the deep statistical diagnostics layer only."""
+    from framevitals.api import statistics as _statistics
+
+    return _statistics(data, max_pairs=max_pairs)
+
+
+def anomalies(
+    data: Any,
+    *,
+    contamination: float = 0.05,
+    threshold: float = 0.6,
+    max_columns: int = 30,
+    top_k: int = 25,
+) -> dict[str, Any]:
+    """Run only the FrameVitals tabular anomaly ensemble."""
+    from framevitals.api import anomalies as _anomalies
+
+    return _anomalies(
+        data,
+        contamination=contamination,
+        threshold=threshold,
+        max_columns=max_columns,
+        top_k=top_k,
+    )
+
+
+def target_analysis(data: Any, *, target: str) -> dict[str, Any]:
+    """Run target quality, leakage, association, and split diagnostics only."""
+    from framevitals.api import target_analysis as _target_analysis
+
+    return _target_analysis(data, target=target)
+
+
 def analyze(
     data: Any,
     *,
@@ -157,6 +237,14 @@ def validate(data: Any, contract: dict[str, Any]) -> ValidationResult:
     "ColumnResult",
     "DriftResult",
     "ValidationResult",
+    "profile",
+    "roles",
+    "health",
+    "ml_readiness",
+    "quality",
+    "statistics",
+    "anomalies",
+    "target_analysis",
     "analyze",
     "plan",
     "plan_cleaning",

From 9a3546fd4a4a35cdd60dd0a9bbae8a74a13740b4 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:18:41 +0530
Subject: [PATCH 089/439] test: cover focused public analysis APIs

---
 tests/test_focused_api.py | 123 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 123 insertions(+)
 create mode 100644 tests/test_focused_api.py

diff --git a/tests/test_focused_api.py b/tests/test_focused_api.py
new file mode 100644
index 0000000..97a5efa
--- /dev/null
+++ b/tests/test_focused_api.py
@@ -0,0 +1,123 @@
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+import framevitals as fv
+from framevitals.health_score import calculate_health_score
+from framevitals.profiler import build_profile
+
+
+def _frame(rows: int = 40) -> pd.DataFrame:
+    return pd.DataFrame({
+        "age": list(range(20, 20 + rows)),
+        "income": [30_000 + index * 1_000 for index in range(rows)],
+        "city": ["Pune", "Mumbai"] * (rows // 2),
+        "target": [0, 1] * (rows // 2),
+    })
+
+
+def test_profile_is_focused_and_matches_profiler():
+    df = _frame()
+
+    result = fv.profile(df)
+    expected = build_profile(df)
+
+    assert result["dataset_name"] == "<dataframe>"
+    assert result["shape"] == expected["shape"]
+    assert result["dtypes"] == expected["dtypes"]
+    assert result["missing_counts"] == expected["missing_counts"]
+
+
+def test_roles_health_and_ml_readiness_are_public():
+    df = _frame()
+
+    role_result = fv.roles(df)
+    health_result = fv.health(df)
+    readiness_result = fv.ml_readiness(df)
+
+    assert role_result["dataset_name"] == "<dataframe>"
+    assert set(role_result["columns"]) == set(df.columns)
+    assert "summary" in role_result
+
+    expected_health = calculate_health_score(df, build_profile(df))
+    assert health_result["overall_score"] == expected_health["overall_score"]
+    assert health_result["label"] == expected_health["label"]
+
+    assert readiness_result["dataset_name"] == "<dataframe>"
+    assert 0 <= readiness_result["score"] <= 100
+
+
+def test_quality_statistics_and_anomalies_can_run_independently():
+    df = _frame()
+
+    quality = fv.quality(df, max_sample_rows=20)
+    statistics = fv.statistics(df, max_pairs=5)
+    anomalies = fv.anomalies(df, top_k=5)
+
+    assert quality["dataset_name"] == "<dataframe>"
+    assert quality["available"] is True
+    assert quality["max_sample_rows"] == 20
+
+    assert statistics["dataset_name"] == "<dataframe>"
+    assert isinstance(statistics, dict)
+
+    assert anomalies["dataset_name"] == "<dataframe>"
+    assert anomalies["available"] is True
+    assert len(anomalies["top_rows"]) <= 5
+
+
+def test_target_analysis_is_focused_and_validates_target():
+    df = _frame()
+
+    result = fv.target_analysis(df, target="target")
+
+    assert result["dataset_name"] == "<dataframe>"
+    assert result["available"] is True
+    assert result["target_column"] == "target"
+    assert result["task_type"] == "classification"
+
+    with pytest.raises(ValueError, match="Target column not found"):
+        fv.target_analysis(df, target="does_not_exist")
+
+
+def test_focused_apis_preserve_file_source_name(tmp_path):
+    path = tmp_path / "customers.csv"
+    _frame().to_csv(path, index=False)
+
+    assert fv.profile(path)["dataset_name"] == "customers.csv"
+    assert fv.quality(path)["dataset_name"] == "customers.csv"
+    assert fv.anomalies(path)["dataset_name"] == "customers.csv"
+
+
+def test_focused_apis_are_side_effect_free(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    df = _frame()
+
+    fv.profile(df)
+    fv.roles(df)
+    fv.health(df)
+    fv.ml_readiness(df)
+    fv.quality(df)
+    fv.statistics(df, max_pairs=5)
+    fv.anomalies(df, top_k=5)
+    fv.target_analysis(df, target="target")
+
+    assert not Path("cleaned").exists()
+    assert not Path("static/charts").exists()
+    assert not Path("reports").exists()
+
+
+def test_focused_functions_are_exposed_from_top_level():
+    for name in (
+        "profile",
+        "roles",
+        "health",
+        "ml_readiness",
+        "quality",
+        "statistics",
+        "anomalies",
+        "target_analysis",
+    ):
+        assert callable(getattr(fv, name))
+        assert name in fv.__all__

From 085f38a3d36391c64cdd11961b7d2594836c92c2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:19:39 +0530
Subject: [PATCH 090/439] feat: add combined quality gate result

---
 src/framevitals/quality_results.py | 48 +++++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/quality_results.py b/src/framevitals/quality_results.py
index 8573977..af9d967 100644
--- a/src/framevitals/quality_results.py
+++ b/src/framevitals/quality_results.py
@@ -1,4 +1,4 @@
-"""Dict-compatible public results for validation and drift workflows."""
+"""Dict-compatible public results for validation, drift, and quality gates."""
 
 from __future__ import annotations
 
@@ -129,3 +129,49 @@ def summary_text(self) -> str:
                     f"- [{str(entry.get('drift_severity')).upper()}] {entry.get('column')}"
                 )
         return "\n".join(lines)
+
+
+class GateResult(_QualityResult):
+    """Combined contract/drift quality-gate result."""
+
+    @property
+    def status(self) -> str:
+        return str(self.get("status", "unknown"))
+
+    @property
+    def passed(self) -> bool:
+        return bool(self.get("passed", False))
+
+    @property
+    def reasons(self) -> list[str]:
+        value = self.get("reasons", [])
+        return value if isinstance(value, list) else []
+
+    def summary_text(self) -> str:
+        checks = self.get("checks", {})
+        if not isinstance(checks, dict):
+            checks = {}
+        validation = checks.get("validation")
+        drift = checks.get("drift")
+
+        lines = [
+            "FrameVitals quality gate",
+            f"Status          {self.status.upper()}",
+            f"Passed          {'YES' if self.passed else 'NO'}",
+        ]
+        if isinstance(validation, dict):
+            lines.append(
+                f"Validation      {str(validation.get('status', 'unknown')).upper()}"
+            )
+        if isinstance(drift, dict):
+            drift_gate = drift.get("gate", {})
+            if isinstance(drift_gate, dict):
+                lines.append(
+                    "Drift           "
+                    f"{str(drift_gate.get('severity', 'unknown')).upper()}"
+                )
+        if self.reasons:
+            lines.extend(["", "Reasons"])
+            for reason in self.reasons[:10]:
+                lines.append(f"- {reason}")
+        return "\n".join(lines)

From cd8a0fb3d09c1c9407ad66f9a0b122193db60105 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:20:29 +0530
Subject: [PATCH 091/439] feat: add combined validation and drift quality gate

---
 src/framevitals/api.py | 130 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 128 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index 3aae427..0e60409 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -20,7 +20,7 @@
 from framevitals.contracts import validate_contract
 from framevitals.dataset_signals import detect_dataset_signals
 from framevitals.deep_statistics_v2 import run_deep_statistics_v2
-from framevitals.drift_analysis import compare_datasets
+from framevitals.drift_analysis import compare_datasets, severity_at_least
 from framevitals.health_score import calculate_health_score
 from framevitals.loader import load_dataset
 from framevitals.ml_readiness import calculate_ml_readiness
@@ -28,11 +28,12 @@
 from framevitals.planning import AnalysisPlan
 from framevitals.profiler import build_profile
 from framevitals.quality_diagnostics import run_quality_diagnostics
-from framevitals.quality_results import DriftResult, ValidationResult
+from framevitals.quality_results import DriftResult, GateResult, ValidationResult
 from framevitals.result import AnalysisResult
 from framevitals.target_intelligence import run_target_intelligence
 
 DataInput = str | Path | pd.DataFrame
+_DRIFT_SEVERITY_RANK = {"stable": 0, "minor": 1, "moderate": 2, "severe": 3}
 
 
 def _validated_path(value: str | Path, *, label: str = "Dataset") -> Path:
@@ -369,3 +370,128 @@ def validate(
     payload = validate_contract(dataframe, contract)
     payload["dataset_name"] = source_name
     return ValidationResult(payload)
+
+
+def gate(
+    current: DataInput,
+    *,
+    reference: DataInput | None = None,
+    contract: Mapping[str, Any] | None = None,
+    columns: list[str] | None = None,
+    max_columns: int = 30,
+    drift_warn_on: str = "moderate",
+    drift_fail_on: str = "severe",
+    fail_on_validation_warning: bool = False,
+) -> GateResult:
+    """Run lightweight validation/drift checks and return one quality verdict.
+
+    At least one of ``reference`` or ``contract`` must be supplied. The current
+    dataset is loaded only once, and no full EDA/model pipeline is executed.
+    Validation failures always fail the gate. Warning-only validation results
+    warn by default and can be promoted to failures. Drift uses independent
+    warning/failure thresholds.
+    """
+    if reference is None and contract is None:
+        raise ValueError("gate requires at least one of reference= or contract=.")
+    if drift_warn_on not in _DRIFT_SEVERITY_RANK:
+        raise ValueError("drift_warn_on must be one of: stable, minor, moderate, severe.")
+    if drift_fail_on not in _DRIFT_SEVERITY_RANK:
+        raise ValueError("drift_fail_on must be one of: stable, minor, moderate, severe.")
+    if _DRIFT_SEVERITY_RANK[drift_warn_on] > _DRIFT_SEVERITY_RANK[drift_fail_on]:
+        raise ValueError("drift_warn_on cannot be more severe than drift_fail_on.")
+    if max_columns < 1:
+        raise ValueError("max_columns must be at least 1.")
+
+    current_df, current_name = _load_input(current, label="Current")
+    checks: dict[str, Any] = {}
+    reasons: list[str] = []
+    status = "pass"
+
+    def warn() -> None:
+        nonlocal status
+        if status == "pass":
+            status = "warn"
+
+    def fail() -> None:
+        nonlocal status
+        status = "fail"
+
+    if contract is not None:
+        validation = validate_contract(current_df, contract)
+        validation["dataset_name"] = current_name
+        checks["validation"] = validation
+
+        validation_status = validation.get("status")
+        if validation_status == "fail":
+            fail()
+            error_count = validation.get("summary", {}).get("errors", 0)
+            reasons.append(f"Contract validation failed with {error_count} error(s).")
+        elif validation_status == "warn":
+            warning_count = validation.get("summary", {}).get("warnings", 0)
+            if fail_on_validation_warning:
+                fail()
+                reasons.append(
+                    f"Contract validation produced {warning_count} warning(s), promoted to failure."
+                )
+            else:
+                warn()
+                reasons.append(f"Contract validation produced {warning_count} warning(s).")
+
+    if reference is not None:
+        reference_df, reference_name = _load_input(reference, label="Reference")
+        drift_payload = compare_datasets(
+            reference_df,
+            current_df,
+            columns=columns,
+            max_columns=max_columns,
+        )
+        drift_payload["reference_name"] = reference_name
+        drift_payload["current_name"] = current_name
+        checks["drift"] = drift_payload
+
+        if not drift_payload.get("available"):
+            fail()
+            reasons.append(
+                "Drift comparison was requested but could not produce a comparable result: "
+                f"{drift_payload.get('reason', 'unknown reason')}."
+            )
+        else:
+            drift_severity = str(
+                drift_payload.get("gate", {}).get("severity", "unknown")
+            )
+            if drift_severity not in _DRIFT_SEVERITY_RANK:
+                fail()
+                reasons.append(
+                    "Drift comparison did not produce a recognized severity verdict."
+                )
+            elif severity_at_least(drift_severity, drift_fail_on):
+                fail()
+                reasons.append(
+                    f"Drift severity {drift_severity} reached fail threshold {drift_fail_on}."
+                )
+            elif severity_at_least(drift_severity, drift_warn_on):
+                warn()
+                reasons.append(
+                    f"Drift severity {drift_severity} reached warning threshold {drift_warn_on}."
+                )
+
+            drift_reasons = drift_payload.get("gate", {}).get("reasons", [])
+            if isinstance(drift_reasons, list):
+                for reason in drift_reasons[:10]:
+                    text = str(reason)
+                    if text and text not in reasons:
+                        reasons.append(text)
+
+    return GateResult({
+        "status": status,
+        "passed": status != "fail",
+        "current_name": current_name,
+        "checks_run": list(checks),
+        "thresholds": {
+            "drift_warn_on": drift_warn_on,
+            "drift_fail_on": drift_fail_on,
+            "fail_on_validation_warning": bool(fail_on_validation_warning),
+        },
+        "reasons": reasons,
+        "checks": checks,
+    })

From 5c61f5798ce8740f0f0a2f318307f099c3060091 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:21:13 +0530
Subject: [PATCH 092/439] test: cover combined quality gate API

---
 tests/test_gate_api.py | 123 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 123 insertions(+)
 create mode 100644 tests/test_gate_api.py

diff --git a/tests/test_gate_api.py b/tests/test_gate_api.py
new file mode 100644
index 0000000..70b4053
--- /dev/null
+++ b/tests/test_gate_api.py
@@ -0,0 +1,123 @@
+import pandas as pd
+import pytest
+
+import framevitals as fv
+
+
+def _reference() -> pd.DataFrame:
+    return pd.DataFrame({
+        "age": list(range(20, 50)),
+        "plan": ["basic", "pro", "team"] * 10,
+    })
+
+
+def test_gate_requires_at_least_one_check():
+    with pytest.raises(ValueError, match="reference=.*contract"):
+        fv.gate(_reference())
+
+
+def test_validation_only_gate_passes_clean_data():
+    reference = _reference()
+    contract = fv.infer_contract(reference)
+
+    result = fv.gate(reference.copy(), contract=contract)
+
+    assert isinstance(result, fv.GateResult)
+    assert isinstance(result, dict)
+    assert result.status == "pass"
+    assert result.passed is True
+    assert result["checks_run"] == ["validation"]
+    assert result["checks"]["validation"]["status"] == "pass"
+    assert "FrameVitals quality gate" in result.summary_text()
+
+
+def test_validation_warning_can_warn_or_be_promoted_to_failure():
+    reference = _reference()
+    contract = fv.infer_contract(reference)
+    current = reference.copy()
+    current.loc[0, "plan"] = "enterprise"
+
+    warning = fv.gate(current, contract=contract)
+    strict = fv.gate(
+        current,
+        contract=contract,
+        fail_on_validation_warning=True,
+    )
+
+    assert warning.status == "warn"
+    assert warning.passed is True
+    assert strict.status == "fail"
+    assert strict.passed is False
+    assert any("promoted to failure" in reason for reason in strict.reasons)
+
+
+def test_drift_only_gate_uses_configurable_thresholds():
+    reference = pd.DataFrame({"value": list(range(50))})
+    current = pd.DataFrame({"value": list(range(100, 150))})
+
+    default = fv.gate(current, reference=reference)
+    strict = fv.gate(
+        current,
+        reference=reference,
+        drift_warn_on="minor",
+        drift_fail_on="moderate",
+    )
+
+    assert default.status in {"warn", "fail"}
+    assert strict.status == "fail"
+    assert strict.passed is False
+    assert strict["checks_run"] == ["drift"]
+    assert strict["checks"]["drift"]["available"] is True
+
+
+def test_combined_gate_failure_wins_over_warning():
+    reference = _reference()
+    contract = fv.infer_contract(reference, numeric_tolerance=0)
+    current = reference.copy()
+    current["age"] = current["age"] + 100
+    current.loc[0, "plan"] = "enterprise"
+
+    result = fv.gate(
+        current,
+        reference=reference,
+        contract=contract,
+        drift_warn_on="minor",
+        drift_fail_on="moderate",
+    )
+
+    assert result.status == "fail"
+    assert result.passed is False
+    assert set(result["checks_run"]) == {"validation", "drift"}
+    assert result["checks"]["validation"]["status"] in {"warn", "fail"}
+    assert result["checks"]["drift"]["available"] is True
+    assert result.reasons
+
+
+def test_requested_but_unavailable_drift_fails_gate():
+    reference = pd.DataFrame({"left": list(range(30))})
+    current = pd.DataFrame({"right": list(range(30))})
+
+    result = fv.gate(current, reference=reference)
+
+    assert result.status == "fail"
+    assert result.passed is False
+    assert result["checks"]["drift"]["available"] is False
+    assert any("could not produce" in reason for reason in result.reasons)
+
+
+def test_gate_validates_threshold_order_and_max_columns():
+    frame = _reference()
+
+    with pytest.raises(ValueError, match="drift_warn_on"):
+        fv.gate(frame, reference=frame, drift_warn_on="extreme")
+    with pytest.raises(ValueError, match="drift_fail_on"):
+        fv.gate(frame, reference=frame, drift_fail_on="extreme")
+    with pytest.raises(ValueError, match="cannot be more severe"):
+        fv.gate(
+            frame,
+            reference=frame,
+            drift_warn_on="severe",
+            drift_fail_on="moderate",
+        )
+    with pytest.raises(ValueError, match="max_columns"):
+        fv.gate(frame, reference=frame, max_columns=0)

From e33bee8343ee5b5c23393d1b1e70c1e3dcb6c1a2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:23:57 +0530
Subject: [PATCH 093/439] test: preserve lazy top-level public API imports

---
 tests/test_lazy_public_api.py | 41 +++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 tests/test_lazy_public_api.py

diff --git a/tests/test_lazy_public_api.py b/tests/test_lazy_public_api.py
new file mode 100644
index 0000000..09f2079
--- /dev/null
+++ b/tests/test_lazy_public_api.py
@@ -0,0 +1,41 @@
+import subprocess
+import sys
+
+
+def test_top_level_import_keeps_heavy_implementation_lazy():
+    script = r'''
+import sys
+import framevitals
+
+required = (
+    "profile",
+    "roles",
+    "health",
+    "ml_readiness",
+    "quality",
+    "statistics",
+    "anomalies",
+    "target_analysis",
+    "analyze",
+    "compare",
+    "validate",
+    "gate",
+)
+for name in required:
+    assert hasattr(framevitals, name), name
+
+assert "framevitals.api" not in sys.modules
+assert "framevitals.pipeline" not in sys.modules
+assert "framevitals.cleaning_plan" not in sys.modules
+assert "pandas" not in sys.modules
+assert "numpy" not in sys.modules
+'''
+
+    completed = subprocess.run(
+        [sys.executable, "-c", script],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+    assert completed.returncode == 0, completed.stderr

From d36cfff79f21c123b2b7f20154a95a01362a10f6 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:36:34 +0530
Subject: [PATCH 094/439] feat: add adaptive execution budgets

---
 src/framevitals/execution.py | 207 +++++++++++++++++++++++++++++++++++
 1 file changed, 207 insertions(+)
 create mode 100644 src/framevitals/execution.py

diff --git a/src/framevitals/execution.py b/src/framevitals/execution.py
new file mode 100644
index 0000000..fafb215
--- /dev/null
+++ b/src/framevitals/execution.py
@@ -0,0 +1,207 @@
+"""Adaptive execution budgets for FrameVitals analyses.
+
+The budget layer is intentionally backend-agnostic. It describes how much raw
+work an analysis may attempt for a given dataset shape and analysis mode. Native
+Rust, Arrow, CUDA, and distributed backends can consume the same contract later.
+
+The immediate goal is safety: expensive statistical routines must never infer
+that a 100k-row input means they should allocate O(n^2) intermediates. Sampling
+is deterministic and must be disclosed in result metadata.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+
+_VALID_MODES = {"quick", "standard", "deep", "research"}
+
+
+@dataclass(frozen=True, slots=True)
+class ExecutionBudget:
+    """Resolved resource policy for one analysis run.
+
+    Counts are upper bounds, not promises that every analysis consumes the full
+    allowance. Backends are encouraged to stop earlier when an estimate has
+    converged or an operation is not applicable.
+    """
+
+    mode: str
+    rows: int
+    columns: int
+    cells: int
+    scale_class: str
+    large_dataset: bool
+    wide_dataset: bool
+    ultra_wide_dataset: bool
+    quality_sample_rows: int
+    deep_statistics_sample_rows: int
+    bootstrap_sample_rows: int
+    distribution_sample_rows: int
+    pair_sample_rows: int
+    anomaly_sample_rows: int
+    time_series_sample_rows: int
+    relationship_pair_budget: int
+    max_memory_heavy_parallelism: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+def _bounded(requested: int, rows: int) -> int:
+    if rows <= 0:
+        return 0
+    return min(int(requested), int(rows))
+
+
+def derive_execution_budget(
+    rows: int,
+    columns: int,
+    *,
+    mode: str = "standard",
+) -> ExecutionBudget:
+    """Derive a conservative execution policy from shape and analysis mode.
+
+    The thresholds are deliberately simple and deterministic for now. They are
+    a compatibility layer for the future cost-based planner, where RAM, storage
+    metadata, native throughput, GPU availability, and user accuracy budgets can
+    refine the same object without changing analysis APIs.
+    """
+    if mode not in _VALID_MODES:
+        raise ValueError(f"Unknown analysis mode: {mode}")
+    if rows < 0 or columns < 0:
+        raise ValueError("rows and columns must be non-negative.")
+
+    cells = int(rows) * int(columns)
+    large_dataset = rows >= 100_000 or cells >= 10_000_000
+    wide_dataset = columns >= 1_000
+    ultra_wide_dataset = columns >= 10_000
+
+    if rows >= 100_000_000 or columns >= 100_000 or cells >= 1_000_000_000_000:
+        scale_class = "extreme"
+    elif rows >= 10_000_000 or columns >= 10_000 or cells >= 10_000_000_000:
+        scale_class = "very_large"
+    elif large_dataset or wide_dataset:
+        scale_class = "large"
+    else:
+        scale_class = "normal"
+
+    presets = {
+        "quick": {
+            "quality": 1_000,
+            "deep": 2_000,
+            "bootstrap": 1_000,
+            "distribution": 2_000,
+            "pair": 2_000,
+            "anomaly": 5_000,
+            "time_series": 5_000,
+            "relationships": 10,
+        },
+        "standard": {
+            "quality": 5_000,
+            "deep": 5_000,
+            "bootstrap": 2_500,
+            "distribution": 5_000,
+            "pair": 5_000,
+            "anomaly": 10_000,
+            "time_series": 10_000,
+            "relationships": 20,
+        },
+        "deep": {
+            "quality": 10_000,
+            "deep": 10_000,
+            "bootstrap": 5_000,
+            "distribution": 10_000,
+            "pair": 10_000,
+            "anomaly": 25_000,
+            "time_series": 25_000,
+            "relationships": 50,
+        },
+        "research": {
+            "quality": 20_000,
+            "deep": 20_000,
+            "bootstrap": 10_000,
+            "distribution": 20_000,
+            "pair": 20_000,
+            "anomaly": 50_000,
+            "time_series": 50_000,
+            "relationships": 100,
+        },
+    }
+    selected = presets[mode]
+
+    # Ultra-wide data must spend relationship budget more carefully. The future
+    # sparse feature-graph engine will replace this fixed cap with candidate
+    # generation rather than dense pair enumeration.
+    relationship_budget = int(selected["relationships"])
+    if ultra_wide_dataset:
+        relationship_budget = min(relationship_budget, 10)
+    elif wide_dataset:
+        relationship_budget = min(relationship_budget, 20)
+
+    # Memory-heavy modules should not be launched four-at-a-time simply because
+    # a machine exposes four Python workers. Large inputs default to sequential
+    # heavy execution until the scheduler gains RAM-aware token accounting.
+    heavy_parallelism = 1 if large_dataset or wide_dataset else 2
+
+    return ExecutionBudget(
+        mode=mode,
+        rows=int(rows),
+        columns=int(columns),
+        cells=cells,
+        scale_class=scale_class,
+        large_dataset=large_dataset,
+        wide_dataset=wide_dataset,
+        ultra_wide_dataset=ultra_wide_dataset,
+        quality_sample_rows=_bounded(selected["quality"], rows),
+        deep_statistics_sample_rows=_bounded(selected["deep"], rows),
+        bootstrap_sample_rows=_bounded(selected["bootstrap"], rows),
+        distribution_sample_rows=_bounded(selected["distribution"], rows),
+        pair_sample_rows=_bounded(selected["pair"], rows),
+        anomaly_sample_rows=_bounded(selected["anomaly"], rows),
+        time_series_sample_rows=_bounded(selected["time_series"], rows),
+        relationship_pair_budget=relationship_budget,
+        max_memory_heavy_parallelism=heavy_parallelism,
+    )
+
+
+def deterministic_sample_frame(
+    dataframe: pd.DataFrame,
+    max_rows: int,
+    *,
+    preserve_order: bool = False,
+) -> tuple[pd.DataFrame, dict[str, Any]]:
+    """Return a deterministic bounded view and transparent sampling metadata."""
+    source_rows = int(len(dataframe))
+    if max_rows < 1:
+        raise ValueError("max_rows must be at least 1.")
+
+    if source_rows <= max_rows:
+        return dataframe, {
+            "sampled": False,
+            "source_rows": source_rows,
+            "sample_rows": source_rows,
+            "strategy": "full",
+        }
+
+    # Evenly spaced positions are deterministic, preserve coverage across the
+    # source, and avoid allocating a random permutation proportional to n.
+    positions = np.linspace(0, source_rows - 1, num=max_rows, dtype=np.int64)
+    positions = np.unique(positions)
+    sampled = dataframe.iloc[positions]
+    if not preserve_order:
+        # The positions are already ordered; this branch documents that callers
+        # may treat the sample as an unordered statistical sample.
+        sampled = sampled.copy()
+
+    return sampled, {
+        "sampled": True,
+        "source_rows": source_rows,
+        "sample_rows": int(len(sampled)),
+        "strategy": "deterministic_evenly_spaced",
+        "preserve_order": bool(preserve_order),
+    }

From 3f9bb846de2b439e3e19519a626a5b8cb7e388e4 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:36:58 +0530
Subject: [PATCH 095/439] feat: add resource-bounded analysis adapters

---
 src/framevitals/budgeted_analysis.py | 152 +++++++++++++++++++++++++++
 1 file changed, 152 insertions(+)
 create mode 100644 src/framevitals/budgeted_analysis.py

diff --git a/src/framevitals/budgeted_analysis.py b/src/framevitals/budgeted_analysis.py
new file mode 100644
index 0000000..9ba5491
--- /dev/null
+++ b/src/framevitals/budgeted_analysis.py
@@ -0,0 +1,152 @@
+"""Resource-bounded adapters around legacy analysis modules.
+
+These adapters are a migration bridge. They make existing analyses safe on
+large inputs today while FrameVitals evolves toward streaming Rust/Arrow kernels
+and per-operation exact/approximate execution. Sampling is deterministic and
+always included in returned metadata.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+
+from framevitals.anomaly_ensemble import detect_anomalies_ensemble
+from framevitals.deep_statistics_v2 import run_deep_statistics_v2
+from framevitals.execution import ExecutionBudget, deterministic_sample_frame
+from framevitals.time_series import detect_and_analyze_time_series
+
+
+def _attach_execution(
+    payload: dict[str, Any],
+    *,
+    budget: ExecutionBudget,
+    sampling: dict[str, Any],
+    scope: str,
+) -> dict[str, Any]:
+    result = dict(payload)
+    result["execution"] = {
+        "scope": scope,
+        "scale_class": budget.scale_class,
+        **sampling,
+    }
+    return result
+
+
+def run_budgeted_deep_statistics(
+    dataframe: pd.DataFrame,
+    *,
+    budget: ExecutionBudget,
+    max_pairs: int | None = None,
+) -> dict[str, Any]:
+    """Run legacy deep statistics without exposing it to unbounded row counts.
+
+    The current v2 implementation performs several operations over one shared
+    frame, including SciPy BCa bootstrap. Until those operations are separated,
+    the strictest relevant row budget is used for the shared frame.
+    """
+    sample_limit = max(
+        20,
+        min(
+            budget.deep_statistics_sample_rows,
+            budget.bootstrap_sample_rows,
+        ),
+    )
+    sample_limit = min(sample_limit, max(len(dataframe), 1))
+    work, sampling = deterministic_sample_frame(dataframe, sample_limit)
+
+    pair_budget = (
+        budget.relationship_pair_budget
+        if max_pairs is None
+        else min(int(max_pairs), budget.relationship_pair_budget)
+    )
+    if pair_budget < 1:
+        raise ValueError("max_pairs must be at least 1.")
+
+    payload = run_deep_statistics_v2(work, max_pairs=pair_budget)
+    sampling = {
+        **sampling,
+        "reason": (
+            "Legacy deep-statistics operations share one bounded frame; "
+            "the strict bootstrap budget is applied to prevent quadratic BCa memory growth."
+            if sampling["sampled"]
+            else "Full input fits within the deep-statistics execution budget."
+        ),
+        "pair_budget": int(pair_budget),
+    }
+    return _attach_execution(
+        payload,
+        budget=budget,
+        sampling=sampling,
+        scope="bounded_deep_statistics",
+    )
+
+
+def run_budgeted_anomalies(
+    dataframe: pd.DataFrame,
+    *,
+    budget: ExecutionBudget,
+    contamination: float = 0.05,
+    threshold: float = 0.6,
+    max_columns: int = 30,
+    top_k: int = 25,
+) -> dict[str, Any]:
+    """Run the anomaly ensemble on a bounded representative frame when needed."""
+    sample_limit = max(20, min(budget.anomaly_sample_rows, max(len(dataframe), 1)))
+    work, sampling = deterministic_sample_frame(dataframe, sample_limit)
+    payload = detect_anomalies_ensemble(
+        work,
+        contamination=contamination,
+        threshold=threshold,
+        max_columns=max_columns,
+        top_k=top_k,
+    )
+    sampling = {
+        **sampling,
+        "reason": (
+            "Expensive neighbor/covariance detectors are bounded until the native "
+            "candidate-filtering anomaly engine is available."
+            if sampling["sampled"]
+            else "Full input fits within the anomaly execution budget."
+        ),
+        "coverage": "sample" if sampling["sampled"] else "full",
+    }
+    return _attach_execution(
+        payload,
+        budget=budget,
+        sampling=sampling,
+        scope="bounded_anomaly_detection",
+    )
+
+
+def run_budgeted_time_series(
+    dataframe: pd.DataFrame,
+    *,
+    budget: ExecutionBudget,
+    target_column: str | None = None,
+) -> dict[str, Any]:
+    """Run time-series discovery/diagnostics on an order-preserving bounded frame."""
+    sample_limit = max(30, min(budget.time_series_sample_rows, max(len(dataframe), 1)))
+    work, sampling = deterministic_sample_frame(
+        dataframe,
+        sample_limit,
+        preserve_order=True,
+    )
+    payload = detect_and_analyze_time_series(work, target_column=target_column)
+    sampling = {
+        **sampling,
+        "reason": (
+            "Time-series diagnostics use an order-preserving bounded view to avoid "
+            "unbounded date parsing, stationarity, PACF, STL, and forecasting work."
+            if sampling["sampled"]
+            else "Full input fits within the time-series execution budget."
+        ),
+        "temporal_order_preserved": True,
+    }
+    return _attach_execution(
+        payload,
+        budget=budget,
+        sampling=sampling,
+        scope="bounded_time_series",
+    )

From 39cc445681c84eed46fc28a97ac9a75d9ec00d49 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:37:42 +0530
Subject: [PATCH 096/439] feat: enforce adaptive execution budgets in pipeline

---
 src/framevitals/pipeline.py | 49 ++++++++++++++++++++++++++++---------
 1 file changed, 37 insertions(+), 12 deletions(-)

diff --git a/src/framevitals/pipeline.py b/src/framevitals/pipeline.py
index 9bb1d39..c655335 100644
--- a/src/framevitals/pipeline.py
+++ b/src/framevitals/pipeline.py
@@ -3,8 +3,8 @@
 Runs the analytics stack in phases and parallelizes independent analyses where
 safe. Optional failures are converted into structured error payloads so a
 single diagnostic does not sink the whole report. Expensive modules can be
-explicitly disabled through ``AnalysisConfig`` while defaults preserve the
-historical execution path.
+explicitly disabled through ``AnalysisConfig`` while adaptive execution budgets
+bound dangerous work on large or wide datasets.
 """
 
 from __future__ import annotations
@@ -20,12 +20,16 @@
 from framevitals.advanced_indicators import calculate_advanced_indicators
 from framevitals.ai_insights import generate_ai_report
 from framevitals.analysis_selector import select_analyses
-from framevitals.anomaly_ensemble import detect_anomalies_ensemble
+from framevitals.budgeted_analysis import (
+    run_budgeted_anomalies,
+    run_budgeted_deep_statistics,
+    run_budgeted_time_series,
+)
 from framevitals.cleaner import create_cleaned_dataset
 from framevitals.column_roles import infer_column_roles, summarize_roles
 from framevitals.config import VALID_MODULES
 from framevitals.dataset_signals import detect_dataset_signals
-from framevitals.deep_statistics_v2 import run_deep_statistics_v2
+from framevitals.execution import derive_execution_budget
 from framevitals.explainability import explain_winner
 from framevitals.health_score import calculate_health_score
 from framevitals.loader import load_dataset
@@ -36,7 +40,6 @@
 from framevitals.signal_engine import build_signals
 from framevitals.target_intelligence import run_target_intelligence
 from framevitals.text_profile import profile_text_columns
-from framevitals.time_series import detect_and_analyze_time_series
 from framevitals.visualizer import generate_charts
 
 
@@ -119,6 +122,12 @@ def module_enabled(name: str) -> bool:
     if target_column is not None and target_column not in df.columns:
         raise ValueError(f"Target column not found: {target_column}")
 
+    execution_budget = derive_execution_budget(
+        len(df),
+        len(df.columns),
+        mode=analysis_mode,
+    )
+
     t0 = time.perf_counter()
     profile = build_profile(df)
     timings_ms["profile"] = (time.perf_counter() - t0) * 1000
@@ -146,6 +155,7 @@ def module_enabled(name: str) -> bool:
         "disabled": sorted(disabled),
         "enabled": sorted(VALID_MODULES - disabled),
     }
+    analysis_selection["execution_budget"] = execution_budget.to_dict()
     timings_ms["analysis_selection"] = (time.perf_counter() - t0) * 1000
 
     # Phase 2: core quality/readiness plus bounded practical diagnostics.
@@ -162,14 +172,13 @@ def module_enabled(name: str) -> bool:
     timings_ms["advanced"] = (time.perf_counter() - t0) * 1000
 
     if module_enabled("quality_diagnostics"):
-        max_sample_rows = 1_000 if analysis_mode == "quick" else 5_000
         _, quality_diagnostics, quality_elapsed = _safe_call(
             "quality_diagnostics",
             lambda: run_quality_diagnostics(
                 df,
                 profile=profile,
                 column_roles=column_roles,
-                max_sample_rows=max_sample_rows,
+                max_sample_rows=max(execution_budget.quality_sample_rows, 10),
             ),
         )
         timings_ms["quality_diagnostics"] = quality_elapsed
@@ -179,20 +188,30 @@ def module_enabled(name: str) -> bool:
         timings_ms["quality_diagnostics"] = 0.0
         module_status["quality_diagnostics"] = "disabled"
 
-    # Phase 3: independent heavier analyses.
+    # Phase 3: independent heavier analyses. Legacy algorithms are routed
+    # through bounded adapters until their streaming/native replacements land.
     deep_statistics_v2 = None
     anomalies_v2 = None
     time_series_analysis = None
     text_profile = None
 
     phase3_modules: list[tuple[str, str, Callable[[], Any]]] = [
-        ("deep_statistics", "deep_statistics_v2", lambda: run_deep_statistics_v2(df)),
-        ("anomaly_detection", "anomalies_v2", lambda: detect_anomalies_ensemble(df)),
+        (
+            "deep_statistics",
+            "deep_statistics_v2",
+            lambda: run_budgeted_deep_statistics(df, budget=execution_budget),
+        ),
+        (
+            "anomaly_detection",
+            "anomalies_v2",
+            lambda: run_budgeted_anomalies(df, budget=execution_budget),
+        ),
         (
             "time_series",
             "time_series",
-            lambda: detect_and_analyze_time_series(
+            lambda: run_budgeted_time_series(
                 df,
+                budget=execution_budget,
                 target_column=target_column,
             ),
         ),
@@ -200,6 +219,10 @@ def module_enabled(name: str) -> bool:
     ]
 
     phase3_results: dict[str, Any] = {}
+    phase3_worker_limit = min(
+        parallel_workers,
+        execution_budget.max_memory_heavy_parallelism,
+    )
     if analysis_mode in {"standard", "deep", "research"}:
         tasks: list[tuple[str, str, Callable[[], Any]]] = []
         for module, result_key, fn in phase3_modules:
@@ -213,7 +236,7 @@ def module_enabled(name: str) -> bool:
         per_task_ms: dict[str, float] = {}
         if tasks:
             phase3_start = time.perf_counter()
-            with ThreadPoolExecutor(max_workers=parallel_workers) as executor:
+            with ThreadPoolExecutor(max_workers=phase3_worker_limit) as executor:
                 futures = {
                     executor.submit(_safe_call, result_key, fn): (module, result_key)
                     for module, result_key, fn in tasks
@@ -433,6 +456,8 @@ def module_enabled(name: str) -> bool:
         "execution": {
             "disabled_modules": sorted(disabled),
             "module_status": module_status,
+            "budget": execution_budget.to_dict(),
+            "phase3_worker_limit": int(phase3_worker_limit),
         },
         "profile": profile,
         "column_roles": column_roles,

From 0ec59ed1bd3f086f08bde68ae8022d1014b8c455 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:38:04 +0530
Subject: [PATCH 097/439] feat: add dataset source abstraction

---
 src/framevitals/sources.py | 113 +++++++++++++++++++++++++++++++++++++
 1 file changed, 113 insertions(+)
 create mode 100644 src/framevitals/sources.py

diff --git a/src/framevitals/sources.py b/src/framevitals/sources.py
new file mode 100644
index 0000000..092486a
--- /dev/null
+++ b/src/framevitals/sources.py
@@ -0,0 +1,113 @@
+"""Dataset source abstraction for FrameVitals.
+
+Today the execution pipeline still materializes supported files through pandas.
+This module establishes the source contract needed for future Arrow/DataFusion
+streaming without changing the public ``fv.analyze(data)`` API.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+import pandas as pd
+
+from framevitals.loader import load_dataset
+
+
+@dataclass(frozen=True, slots=True)
+class DatasetMetadata:
+    """Cheap source metadata available before deep analysis."""
+
+    name: str
+    kind: str
+    format: str
+    rows: int | None
+    columns: int | None
+    size_bytes: int | None
+    materialized: bool
+    supports_projection: bool
+    supports_streaming: bool
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@runtime_checkable
+class DatasetSource(Protocol):
+    """Minimal contract implemented by FrameVitals data sources."""
+
+    def inspect(self) -> DatasetMetadata: ...
+
+    def load(self) -> pd.DataFrame: ...
+
+
+@dataclass(slots=True)
+class PandasSource:
+    dataframe: pd.DataFrame
+    name: str = "<dataframe>"
+
+    def inspect(self) -> DatasetMetadata:
+        rows, columns = self.dataframe.shape
+        size_bytes = int(self.dataframe.memory_usage(index=True, deep=True).sum())
+        return DatasetMetadata(
+            name=self.name,
+            kind="memory",
+            format="pandas",
+            rows=int(rows),
+            columns=int(columns),
+            size_bytes=size_bytes,
+            materialized=True,
+            supports_projection=True,
+            supports_streaming=False,
+        )
+
+    def load(self) -> pd.DataFrame:
+        if self.dataframe.empty:
+            raise ValueError("Dataset DataFrame is empty.")
+        return self.dataframe.copy()
+
+
+@dataclass(slots=True)
+class FileSource:
+    path: Path
+
+    def inspect(self) -> DatasetMetadata:
+        if not self.path.exists():
+            raise FileNotFoundError(f"Dataset not found: {self.path}")
+        if not self.path.is_file():
+            raise ValueError(f"Expected a dataset file, got: {self.path}")
+
+        suffix = self.path.suffix.lower().lstrip(".") or "unknown"
+        # The current loader materializes every supported file. These capability
+        # flags describe what FrameVitals can do *today*, not what the format
+        # could theoretically support. Arrow/Parquet sources will override them.
+        return DatasetMetadata(
+            name=self.path.name,
+            kind="file",
+            format=suffix,
+            rows=None,
+            columns=None,
+            size_bytes=int(self.path.stat().st_size),
+            materialized=False,
+            supports_projection=False,
+            supports_streaming=False,
+        )
+
+    def load(self) -> pd.DataFrame:
+        dataframe = load_dataset(self.path)
+        if dataframe.empty:
+            raise ValueError(f"Dataset is empty: {self.path}")
+        return dataframe
+
+
+def resolve_source(data: str | Path | pd.DataFrame | DatasetSource) -> DatasetSource:
+    """Normalize supported user inputs into a DatasetSource implementation."""
+    if isinstance(data, pd.DataFrame):
+        return PandasSource(data)
+    if isinstance(data, (str, Path)):
+        return FileSource(Path(data))
+    if isinstance(data, DatasetSource):
+        return data
+    raise TypeError("data must be a pandas DataFrame, dataset path, or DatasetSource.")

From f3484da7e6f817e299c1c2359c4c0a6ce88424a2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:38:21 +0530
Subject: [PATCH 098/439] feat: add mergeable analysis state primitives

---
 src/framevitals/analysis_state.py | 175 ++++++++++++++++++++++++++++++
 1 file changed, 175 insertions(+)
 create mode 100644 src/framevitals/analysis_state.py

diff --git a/src/framevitals/analysis_state.py b/src/framevitals/analysis_state.py
new file mode 100644
index 0000000..c729ba7
--- /dev/null
+++ b/src/framevitals/analysis_state.py
@@ -0,0 +1,175 @@
+"""Mergeable analysis-state primitives.
+
+These classes are the reference semantics for FrameVitals' future Rust/Arrow
+streaming engine. A partition can be summarized independently and merged with
+another partition without concatenating raw rows.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from math import sqrt
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+
+@dataclass(slots=True)
+class NumericColumnState:
+    """Mergeable exact first/second-moment state for one numeric column."""
+
+    count: int = 0
+    missing: int = 0
+    mean: float = 0.0
+    m2: float = 0.0
+    minimum: float | None = None
+    maximum: float | None = None
+    infinite: int = 0
+
+    @classmethod
+    def from_series(cls, series: pd.Series) -> "NumericColumnState":
+        values = pd.to_numeric(series, errors="coerce").to_numpy(
+            dtype="float64",
+            na_value=np.nan,
+        )
+        missing = int(np.isnan(values).sum())
+        infinite = int(np.isinf(values).sum())
+        finite = values[np.isfinite(values)]
+        if finite.size == 0:
+            return cls(missing=missing, infinite=infinite)
+
+        mean = float(finite.mean())
+        centered = finite - mean
+        return cls(
+            count=int(finite.size),
+            missing=missing,
+            mean=mean,
+            m2=float(np.dot(centered, centered)),
+            minimum=float(finite.min()),
+            maximum=float(finite.max()),
+            infinite=infinite,
+        )
+
+    def merge(self, other: "NumericColumnState") -> "NumericColumnState":
+        """Merge another partition using the parallel-variance formula."""
+        if other.count == 0:
+            return NumericColumnState(
+                count=self.count,
+                missing=self.missing + other.missing,
+                mean=self.mean,
+                m2=self.m2,
+                minimum=self.minimum,
+                maximum=self.maximum,
+                infinite=self.infinite + other.infinite,
+            )
+        if self.count == 0:
+            return NumericColumnState(
+                count=other.count,
+                missing=self.missing + other.missing,
+                mean=other.mean,
+                m2=other.m2,
+                minimum=other.minimum,
+                maximum=other.maximum,
+                infinite=self.infinite + other.infinite,
+            )
+
+        total = self.count + other.count
+        delta = other.mean - self.mean
+        mean = self.mean + delta * other.count / total
+        m2 = (
+            self.m2
+            + other.m2
+            + delta * delta * self.count * other.count / total
+        )
+        minimum = min(
+            value for value in (self.minimum, other.minimum) if value is not None
+        )
+        maximum = max(
+            value for value in (self.maximum, other.maximum) if value is not None
+        )
+        return NumericColumnState(
+            count=total,
+            missing=self.missing + other.missing,
+            mean=float(mean),
+            m2=float(m2),
+            minimum=float(minimum),
+            maximum=float(maximum),
+            infinite=self.infinite + other.infinite,
+        )
+
+    @property
+    def variance(self) -> float | None:
+        if self.count < 2:
+            return None
+        return self.m2 / (self.count - 1)
+
+    @property
+    def std(self) -> float | None:
+        variance = self.variance
+        return sqrt(variance) if variance is not None and variance >= 0 else None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "count": self.count,
+            "missing": self.missing,
+            "infinite": self.infinite,
+            "mean": self.mean if self.count else None,
+            "variance": self.variance,
+            "std": self.std,
+            "minimum": self.minimum,
+            "maximum": self.maximum,
+        }
+
+
+@dataclass(slots=True)
+class AnalysisState:
+    """Compact mergeable state for a dataset partition."""
+
+    rows: int = 0
+    columns: int = 0
+    numeric: dict[str, NumericColumnState] = field(default_factory=dict)
+    schema: dict[str, str] = field(default_factory=dict)
+
+    @classmethod
+    def from_frame(cls, dataframe: pd.DataFrame) -> "AnalysisState":
+        numeric_columns = dataframe.select_dtypes(include=[np.number]).columns
+        return cls(
+            rows=int(len(dataframe)),
+            columns=int(len(dataframe.columns)),
+            numeric={
+                str(column): NumericColumnState.from_series(dataframe[column])
+                for column in numeric_columns
+            },
+            schema={str(column): str(dtype) for column, dtype in dataframe.dtypes.items()},
+        )
+
+    def merge(self, other: "AnalysisState") -> "AnalysisState":
+        """Merge states without access to either partition's raw rows."""
+        if self.schema and other.schema and self.schema != other.schema:
+            raise ValueError("Cannot merge AnalysisState objects with different schemas.")
+
+        names = set(self.numeric) | set(other.numeric)
+        merged_numeric: dict[str, NumericColumnState] = {}
+        for name in names:
+            left = self.numeric.get(name, NumericColumnState())
+            right = other.numeric.get(name, NumericColumnState())
+            merged_numeric[name] = left.merge(right)
+
+        return AnalysisState(
+            rows=self.rows + other.rows,
+            columns=max(self.columns, other.columns),
+            numeric=merged_numeric,
+            schema=dict(self.schema or other.schema),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "rows": self.rows,
+            "columns": self.columns,
+            "schema": dict(self.schema),
+            "numeric": {
+                name: state.to_dict()
+                for name, state in sorted(self.numeric.items())
+            },
+        }

From 3a14481f2061994ef6ea4a835cf5e872e14b52ac Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:38:39 +0530
Subject: [PATCH 099/439] fix: clean analysis state imports

---
 src/framevitals/analysis_state.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/framevitals/analysis_state.py b/src/framevitals/analysis_state.py
index c729ba7..890e2ea 100644
--- a/src/framevitals/analysis_state.py
+++ b/src/framevitals/analysis_state.py
@@ -7,7 +7,7 @@
 
 from __future__ import annotations
 
-from dataclasses import asdict, dataclass, field
+from dataclasses import dataclass, field
 from math import sqrt
 from typing import Any
 

From f84e04a4320b44db0bcb38c08bf7268017d02801 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:38:58 +0530
Subject: [PATCH 100/439] test: cover adaptive execution and mergeable state

---
 tests/test_adaptive_execution.py | 150 +++++++++++++++++++++++++++++++
 1 file changed, 150 insertions(+)
 create mode 100644 tests/test_adaptive_execution.py

diff --git a/tests/test_adaptive_execution.py b/tests/test_adaptive_execution.py
new file mode 100644
index 0000000..4c98fc8
--- /dev/null
+++ b/tests/test_adaptive_execution.py
@@ -0,0 +1,150 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from framevitals.analysis_state import AnalysisState, NumericColumnState
+from framevitals.budgeted_analysis import (
+    run_budgeted_anomalies,
+    run_budgeted_deep_statistics,
+    run_budgeted_time_series,
+)
+from framevitals.execution import derive_execution_budget, deterministic_sample_frame
+
+
+def test_large_budget_limits_memory_heavy_parallelism():
+    budget = derive_execution_budget(100_000, 105, mode="standard")
+
+    assert budget.scale_class == "large"
+    assert budget.large_dataset is True
+    assert budget.max_memory_heavy_parallelism == 1
+    assert budget.bootstrap_sample_rows < budget.rows
+    assert budget.time_series_sample_rows < budget.rows
+
+
+def test_extreme_shape_is_classified_without_allocating_dataset():
+    budget = derive_execution_budget(100_000_000, 100_000, mode="standard")
+
+    assert budget.scale_class == "extreme"
+    assert budget.ultra_wide_dataset is True
+    assert budget.cells == 10_000_000_000_000
+    assert budget.max_memory_heavy_parallelism == 1
+
+
+def test_deterministic_sample_is_bounded_and_covers_endpoints():
+    frame = pd.DataFrame({"x": np.arange(10_000)})
+    sampled, metadata = deterministic_sample_frame(frame, 1_000)
+
+    assert len(sampled) == 1_000
+    assert sampled.iloc[0]["x"] == 0
+    assert sampled.iloc[-1]["x"] == 9_999
+    assert metadata["sampled"] is True
+    assert metadata["source_rows"] == 10_000
+    assert metadata["sample_rows"] == 1_000
+
+
+def test_numeric_state_merge_matches_full_frame_statistics():
+    frame = pd.DataFrame({
+        "x": [1.0, 2.0, np.nan, 4.0, np.inf, 7.0, 9.0],
+        "y": [10, 11, 12, 13, 14, 15, 16],
+    })
+
+    full = AnalysisState.from_frame(frame)
+    left = AnalysisState.from_frame(frame.iloc[:3])
+    right = AnalysisState.from_frame(frame.iloc[3:])
+    merged = left.merge(right)
+
+    assert merged.rows == full.rows
+    assert merged.schema == full.schema
+    for name in ("x", "y"):
+        actual = merged.numeric[name]
+        expected = full.numeric[name]
+        assert actual.count == expected.count
+        assert actual.missing == expected.missing
+        assert actual.infinite == expected.infinite
+        assert actual.mean == pytest.approx(expected.mean)
+        assert actual.variance == pytest.approx(expected.variance)
+        assert actual.minimum == expected.minimum
+        assert actual.maximum == expected.maximum
+
+
+def test_numeric_state_rejects_incompatible_schema_merge():
+    left = AnalysisState.from_frame(pd.DataFrame({"x": [1, 2]}))
+    right = AnalysisState.from_frame(pd.DataFrame({"x": [1.0, 2.0]}))
+
+    with pytest.raises(ValueError, match="different schemas"):
+        left.merge(right)
+
+
+def test_deep_statistics_adapter_never_passes_unbounded_frame(monkeypatch):
+    seen = {}
+
+    def fake_deep(frame, max_pairs=20):
+        seen["rows"] = len(frame)
+        seen["pairs"] = max_pairs
+        return {"available": True}
+
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.run_deep_statistics_v2",
+        fake_deep,
+    )
+    frame = pd.DataFrame({"x": np.arange(20_000), "y": np.arange(20_000)})
+    budget = derive_execution_budget(len(frame), len(frame.columns), mode="standard")
+
+    result = run_budgeted_deep_statistics(frame, budget=budget)
+
+    assert seen["rows"] <= budget.bootstrap_sample_rows
+    assert seen["pairs"] <= budget.relationship_pair_budget
+    assert result["execution"]["sampled"] is True
+    assert result["execution"]["source_rows"] == 20_000
+
+
+def test_anomaly_adapter_discloses_sample_coverage(monkeypatch):
+    seen = {}
+
+    def fake_anomalies(frame, **kwargs):
+        seen["rows"] = len(frame)
+        return {"available": True, "n_rows_scored": len(frame)}
+
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.detect_anomalies_ensemble",
+        fake_anomalies,
+    )
+    frame = pd.DataFrame({"x": np.arange(20_000)})
+    budget = derive_execution_budget(len(frame), 1, mode="standard")
+
+    result = run_budgeted_anomalies(frame, budget=budget)
+
+    assert seen["rows"] == budget.anomaly_sample_rows
+    assert result["execution"]["coverage"] == "sample"
+    assert result["execution"]["sample_rows"] == budget.anomaly_sample_rows
+
+
+def test_time_series_adapter_preserves_order(monkeypatch):
+    seen = {}
+
+    def fake_time_series(frame, target_column=None):
+        seen["values"] = frame["x"].tolist()
+        return {"available": True}
+
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.detect_and_analyze_time_series",
+        fake_time_series,
+    )
+    frame = pd.DataFrame({"x": np.arange(20_000)})
+    budget = derive_execution_budget(len(frame), 1, mode="standard")
+
+    result = run_budgeted_time_series(frame, budget=budget)
+
+    assert seen["values"] == sorted(seen["values"])
+    assert result["execution"]["temporal_order_preserved"] is True
+    assert result["execution"]["sample_rows"] == budget.time_series_sample_rows
+
+
+def test_numeric_column_state_empty_values_are_safe():
+    state = NumericColumnState.from_series(pd.Series([np.nan, np.inf, -np.inf]))
+
+    assert state.count == 0
+    assert state.missing == 1
+    assert state.infinite == 2
+    assert state.mean == 0.0
+    assert state.variance is None

From a794236b2489d992ed6bf6bb4bdd22ce26a8286b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:39:30 +0530
Subject: [PATCH 101/439] perf: reduce advanced indicator scans and memory

---
 src/framevitals/advanced_indicators.py | 118 +++++++++++++++++++------
 1 file changed, 92 insertions(+), 26 deletions(-)

diff --git a/src/framevitals/advanced_indicators.py b/src/framevitals/advanced_indicators.py
index f6b7b93..7291a7c 100644
--- a/src/framevitals/advanced_indicators.py
+++ b/src/framevitals/advanced_indicators.py
@@ -1,10 +1,35 @@
+from __future__ import annotations
+
+import re
+
 import numpy as np
 import pandas as pd
 
-SENSITIVE_KEYWORDS = [
-    "gender", "sex", "age", "race", "religion",
-    "region", "location", "income", "salary", "caste",
-]
+SENSITIVE_KEYWORDS = {
+    "gender",
+    "sex",
+    "age",
+    "race",
+    "religion",
+    "region",
+    "location",
+    "income",
+    "salary",
+    "caste",
+}
+
+
+def _column_tokens(name: object) -> set[str]:
+    normalized = re.sub(r"[^a-z0-9]+", "_", str(name).strip().lower()).strip("_")
+    return {token for token in normalized.split("_") if token}
+
+
+def _bounded_non_null_sample(series: pd.Series, max_rows: int = 200) -> pd.Series:
+    clean = series.dropna()
+    if len(clean) <= max_rows:
+        return clean
+    positions = np.linspace(0, len(clean) - 1, num=max_rows, dtype=np.int64)
+    return clean.iloc[np.unique(positions)]
 
 
 def calculate_column_utility(df):
@@ -45,37 +70,65 @@ def calculate_column_utility(df):
 
 
 def calculate_anomalies(df):
+    """Calculate simple IQR anomaly density using O(rows) auxiliary memory."""
     numeric = df.select_dtypes(include=[np.number])
 
     if numeric.empty:
         return {"anomalous_rows": 0, "highest_score": 0, "top_rows": []}
 
-    flags = pd.DataFrame(index=df.index)
+    # The old implementation materialized one boolean column per numeric
+    # feature. A single counter vector produces the identical row score while
+    # avoiding O(rows * numeric_columns) temporary memory.
+    row_hits = np.zeros(len(df), dtype=np.uint32)
+    usable_columns = 0
 
     for col in numeric.columns:
-        q1 = numeric[col].quantile(0.25)
-        q3 = numeric[col].quantile(0.75)
+        series = numeric[col]
+        q1 = series.quantile(0.25)
+        q3 = series.quantile(0.75)
         iqr = q3 - q1
 
         if pd.isna(iqr) or iqr == 0:
-            flags[col] = False
             continue
 
         lower = q1 - 1.5 * iqr
         upper = q3 + 1.5 * iqr
-        flags[col] = (numeric[col] < lower) | (numeric[col] > upper)
+        mask = ((series < lower) | (series > upper)).to_numpy(dtype=bool)
+        row_hits += mask
+        usable_columns += 1
 
-    scores = flags.sum(axis=1) / max(len(numeric.columns), 1)
-    top = scores.sort_values(ascending=False).head(10)
+    denominator = max(usable_columns, 1)
+    scores = row_hits.astype(np.float64) / denominator
+    anomalous_mask = row_hits > 0
+
+    if not anomalous_mask.any():
+        return {"anomalous_rows": 0, "highest_score": 0, "top_rows": []}
+
+    top_count = min(10, len(scores))
+    candidate_positions = np.argpartition(scores, -top_count)[-top_count:]
+    candidate_positions = candidate_positions[
+        np.argsort(scores[candidate_positions])[::-1]
+    ]
+
+    top_rows = []
+    for position in candidate_positions:
+        score = float(scores[position])
+        if score <= 0:
+            continue
+        index_value = df.index[int(position)]
+        top_rows.append({
+            "row_index": (
+                int(index_value)
+                if isinstance(index_value, (int, np.integer))
+                else str(index_value)
+            ),
+            "score": round(score, 3),
+        })
 
     return {
-        "anomalous_rows": int((scores > 0).sum()),
+        "anomalous_rows": int(anomalous_mask.sum()),
         "highest_score": round(float(scores.max()), 3),
-        "top_rows": [
-            {"row_index": int(idx), "score": round(float(score), 3)}
-            for idx, score in top.items()
-            if score > 0
-        ],
+        "top_rows": top_rows,
     }
 
 
@@ -83,8 +136,8 @@ def detect_fairness_review(df):
     found = []
 
     for col in df.columns:
-        lower_col = col.lower()
-        if any(keyword in lower_col for keyword in SENSITIVE_KEYWORDS):
+        tokens = _column_tokens(col)
+        if tokens & SENSITIVE_KEYWORDS:
             found.append(col)
 
     if found:
@@ -102,21 +155,34 @@ def detect_fairness_review(df):
 
 
 def calculate_freshness(df):
-    date_columns = []
+    """Detect a date column with bounded screening and one full parse at most."""
+    candidates: list[tuple[float, str]] = []
 
     for col in df.columns:
-        if pd.api.types.is_numeric_dtype(df[col]):
+        series = df[col]
+        if pd.api.types.is_numeric_dtype(series):
             continue
 
-        parsed = pd.to_datetime(df[col], errors="coerce", format="mixed")
+        sample = _bounded_non_null_sample(series, 200)
+        if sample.empty:
+            continue
+
+        parsed_sample = pd.to_datetime(sample, errors="coerce", format="mixed")
+        parse_rate = float(parsed_sample.notna().mean())
+        if parse_rate >= 0.7:
+            candidates.append((parse_rate, col))
 
-        if parsed.notna().mean() >= 0.7:
-            date_columns.append((col, parsed))
+    if not candidates:
+        return {"available": False, "message": "No strong date column detected."}
 
-    if not date_columns:
+    # Parse only the strongest candidate across the complete column. Future
+    # Arrow/Rust sources will calculate extrema while streaming instead.
+    candidates.sort(key=lambda item: (-item[0], item[1]))
+    _, col = candidates[0]
+    parsed = pd.to_datetime(df[col], errors="coerce", format="mixed")
+    if parsed.notna().mean() < 0.7:
         return {"available": False, "message": "No strong date column detected."}
 
-    col, parsed = date_columns[0]
     min_date = parsed.min()
     max_date = parsed.max()
 

From 65e68a6d6563446eda4d444e098fc1a6d1b1f18c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:39:48 +0530
Subject: [PATCH 102/439] test: cover scalable source and indicator foundations

---
 tests/test_scalable_foundations.py | 90 ++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)
 create mode 100644 tests/test_scalable_foundations.py

diff --git a/tests/test_scalable_foundations.py b/tests/test_scalable_foundations.py
new file mode 100644
index 0000000..8387a52
--- /dev/null
+++ b/tests/test_scalable_foundations.py
@@ -0,0 +1,90 @@
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from framevitals.advanced_indicators import (
+    calculate_anomalies,
+    calculate_freshness,
+    detect_fairness_review,
+)
+from framevitals.sources import FileSource, PandasSource, resolve_source
+
+
+def test_advanced_anomaly_scores_match_expected_iqr_density():
+    frame = pd.DataFrame({
+        "a": [0, 0, 0, 0, 100],
+        "b": [1, 1, 1, 1, 50],
+    })
+
+    result = calculate_anomalies(frame)
+
+    assert result["anomalous_rows"] == 1
+    assert result["highest_score"] == 1.0
+    assert result["top_rows"][0]["row_index"] == 4
+    assert result["top_rows"][0]["score"] == 1.0
+
+
+def test_sensitive_name_matching_uses_tokens_not_substrings():
+    frame = pd.DataFrame({
+        "average_score": [1, 2],
+        "paid_amount": [5, 6],
+        "customer_age": [20, 30],
+    })
+
+    result = detect_fairness_review(frame)
+
+    assert result["needs_review"] is True
+    assert result["columns"] == ["customer_age"]
+
+
+def test_freshness_screens_candidates_before_full_parse():
+    frame = pd.DataFrame({
+        "identifier": [f"item-{i}" for i in range(500)],
+        "event_date": pd.date_range("2025-01-01", periods=500, freq="D").astype(str),
+    })
+
+    result = calculate_freshness(frame)
+
+    assert result["available"] is True
+    assert result["date_column"] == "event_date"
+    assert result["oldest_record"] == "2025-01-01"
+
+
+def test_pandas_source_reports_materialized_metadata():
+    frame = pd.DataFrame({"x": np.arange(10), "label": ["a"] * 10})
+    source = PandasSource(frame)
+
+    metadata = source.inspect()
+
+    assert metadata.rows == 10
+    assert metadata.columns == 2
+    assert metadata.materialized is True
+    assert metadata.supports_projection is True
+    assert source.load().equals(frame)
+    assert source.load() is not frame
+
+
+def test_file_source_reports_cheap_metadata_without_loading(tmp_path):
+    path = tmp_path / "data.csv"
+    path.write_text("x,y\n1,2\n3,4\n", encoding="utf-8")
+    source = resolve_source(path)
+
+    assert isinstance(source, FileSource)
+    metadata = source.inspect()
+    assert metadata.name == "data.csv"
+    assert metadata.format == "csv"
+    assert metadata.rows is None
+    assert metadata.size_bytes == path.stat().st_size
+
+    loaded = source.load()
+    assert loaded.shape == (2, 2)
+
+
+def test_resolve_source_rejects_unsupported_objects():
+    try:
+        resolve_source(object())
+    except TypeError as exc:
+        assert "DatasetSource" in str(exc)
+    else:
+        raise AssertionError("resolve_source should reject unsupported objects")

From f87c64f4af1b3430e711b8b5f5f43ae94923dbe1 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:40:34 +0530
Subject: [PATCH 103/439] fix: make role matching token-aware and bounded

---
 src/framevitals/column_roles.py | 50 +++++++++++++++++++++++++++++----
 1 file changed, 45 insertions(+), 5 deletions(-)

diff --git a/src/framevitals/column_roles.py b/src/framevitals/column_roles.py
index c8d1cd6..7c06f40 100644
--- a/src/framevitals/column_roles.py
+++ b/src/framevitals/column_roles.py
@@ -5,6 +5,11 @@
 ratio, missingness, statistical properties, and bounded value-pattern samples.
 """
 
+from __future__ import annotations
+
+import re
+
+import numpy as np
 import pandas as pd
 
 from framevitals.semantic_types import infer_semantic_types
@@ -66,9 +71,43 @@
 }
 
 
+def _normalise_name(value: object) -> tuple[str, tuple[str, ...]]:
+    normalized = re.sub(r"[^a-z0-9]+", "_", str(value).strip().lower()).strip("_")
+    return normalized, tuple(token for token in normalized.split("_") if token)
+
+
 def _name_matches(column_name: str, keywords: list) -> bool:
-    lower = column_name.lower().replace("-", "_")
-    return any(kw in lower for kw in keywords)
+    """Match keyword tokens/phrases without unsafe substring collisions.
+
+    This prevents names such as ``paid_amount`` from matching ``id`` and
+    ``average_score`` from matching ``age`` while retaining conventional names
+    such as ``customer_id``, ``created_at``, and ``roll_number``.
+    """
+    normalized, tokens = _normalise_name(column_name)
+    token_set = set(tokens)
+
+    for keyword in keywords:
+        keyword_normalized, keyword_tokens = _normalise_name(keyword)
+        if not keyword_normalized:
+            continue
+        if normalized == keyword_normalized:
+            return True
+        if len(keyword_tokens) == 1 and keyword_tokens[0] in token_set:
+            return True
+        if len(keyword_tokens) > 1:
+            width = len(keyword_tokens)
+            for start in range(0, len(tokens) - width + 1):
+                if tokens[start : start + width] == keyword_tokens:
+                    return True
+    return False
+
+
+def _bounded_text_sample(series: pd.Series, max_rows: int = 500) -> pd.Series:
+    clean = series.dropna()
+    if len(clean) <= max_rows:
+        return clean.astype(str)
+    positions = np.linspace(0, len(clean) - 1, num=max_rows, dtype=np.int64)
+    return clean.iloc[np.unique(positions)].astype(str)
 
 
 def _classify_missingness(missing_percent: float) -> str:
@@ -153,15 +192,16 @@ def _infer_single_column_roles(column: str, series: pd.Series, rows: int) -> dic
         roles.add("price_like")
 
     if not is_numeric and not is_bool:
-        sample = series.dropna().astype(str).head(30)
+        sample = _bounded_text_sample(series, 30)
         if len(sample) > 0:
             parsed = pd.to_datetime(sample, errors="coerce", format="mixed")
             if parsed.notna().mean() >= 0.7:
                 roles.add("time_like")
 
     if is_text:
-        lengths = series.dropna().astype(str).str.len()
-        if len(lengths) > 0:
+        sample = _bounded_text_sample(series, 500)
+        if len(sample) > 0:
+            lengths = sample.str.len()
             avg_len = float(lengths.mean())
             max_len = int(lengths.max())
             if avg_len > 50 or max_len > 200:

From ddf5b6e0e1aa5bdb247eb2787045470f73e3a5d9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:40:43 +0530
Subject: [PATCH 104/439] test: guard role keyword boundaries

---
 tests/test_role_keyword_boundaries.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 tests/test_role_keyword_boundaries.py

diff --git a/tests/test_role_keyword_boundaries.py b/tests/test_role_keyword_boundaries.py
new file mode 100644
index 0000000..020ac3b
--- /dev/null
+++ b/tests/test_role_keyword_boundaries.py
@@ -0,0 +1,20 @@
+import pandas as pd
+
+from framevitals.column_roles import infer_column_roles
+
+
+def test_role_keywords_use_token_boundaries():
+    frame = pd.DataFrame({
+        "paid_amount": [10.0, 20.0, 30.0, 40.0],
+        "average_score": [0.1, 0.2, 0.3, 0.4],
+        "customer_id": [101, 102, 103, 104],
+        "customer_age": [20, 21, 22, 23],
+    })
+
+    roles = infer_column_roles(frame)
+
+    assert "id_like" not in roles["paid_amount"]["roles"]
+    assert "price_like" in roles["paid_amount"]["roles"]
+    assert "sensitive" not in roles["average_score"]["roles"]
+    assert "id_like" in roles["customer_id"]["roles"]
+    assert "sensitive" in roles["customer_age"]["roles"]

From 2c684aee85cb6174a71365ee4b9fd0a3c1e53119 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:41:26 +0530
Subject: [PATCH 105/439] perf: bound dense profiler operations

---
 src/framevitals/profiler.py | 102 +++++++++++++++++++++++++++++++-----
 1 file changed, 90 insertions(+), 12 deletions(-)

diff --git a/src/framevitals/profiler.py b/src/framevitals/profiler.py
index 9847edb..4f7a79a 100644
--- a/src/framevitals/profiler.py
+++ b/src/framevitals/profiler.py
@@ -1,6 +1,12 @@
+from __future__ import annotations
+
 import numpy as np
 import pandas as pd
 
+MAX_CORRELATION_COLUMNS = 100
+MAX_EXACT_DUPLICATE_CELLS = 50_000_000
+DUPLICATE_SAMPLE_ROWS = 50_000
+
 
 def clean_value(value):
     if pd.isna(value):
@@ -36,13 +42,91 @@ def detect_column_types(df):
     return numeric_cols, categorical_cols, date_cols
 
 
+def _missing_counts(df: pd.DataFrame) -> pd.Series:
+    """Count missing values without materializing a frame-sized boolean table."""
+    return pd.Series(
+        {column: int(df[column].isna().sum()) for column in df.columns},
+        dtype="int64",
+    )
+
+
+def _duplicate_profile(df: pd.DataFrame) -> tuple[int, dict]:
+    """Return duplicate estimate plus explicit execution metadata."""
+    rows, columns = df.shape
+    cells = rows * columns
+    if cells <= MAX_EXACT_DUPLICATE_CELLS:
+        count = int(df.duplicated().sum())
+        return count, {
+            "method": "exact",
+            "sampled": False,
+            "sample_rows": rows,
+        }
+
+    sample_rows = min(rows, DUPLICATE_SAMPLE_ROWS)
+    positions = np.linspace(0, rows - 1, num=sample_rows, dtype=np.int64)
+    sample = df.iloc[np.unique(positions)]
+    sample_duplicates = int(sample.duplicated().sum())
+    rate = sample_duplicates / max(len(sample), 1)
+    estimate = int(round(rate * rows))
+    return estimate, {
+        "method": "sample_estimate",
+        "sampled": True,
+        "sample_rows": int(len(sample)),
+        "source_rows": int(rows),
+        "estimated_duplicate_rate": round(float(rate), 6),
+    }
+
+
+def _bounded_correlations(
+    df: pd.DataFrame,
+    numeric_cols: list[str],
+) -> tuple[dict, dict]:
+    if len(numeric_cols) < 2:
+        return {}, {
+            "method": "not_applicable",
+            "columns_used": len(numeric_cols),
+            "columns_available": len(numeric_cols),
+            "truncated": False,
+        }
+
+    selected = numeric_cols
+    truncated = len(selected) > MAX_CORRELATION_COLUMNS
+    if truncated:
+        # Prefer columns with the most usable observations. This is a safe
+        # compatibility fallback; the sparse relationship graph will replace
+        # dense truncation for ultra-wide datasets.
+        non_missing = {
+            column: int(df[column].notna().sum())
+            for column in numeric_cols
+        }
+        selected = sorted(
+            numeric_cols,
+            key=lambda column: (-non_missing[column], str(column)),
+        )[:MAX_CORRELATION_COLUMNS]
+
+    correlations = (
+        df[selected]
+        .corr(numeric_only=True)
+        .round(3)
+        .replace({np.nan: None})
+        .to_dict()
+    )
+    return correlations, {
+        "method": "dense_bounded" if truncated else "dense_exact_columns",
+        "columns_used": len(selected),
+        "columns_available": len(numeric_cols),
+        "truncated": truncated,
+        "max_columns": MAX_CORRELATION_COLUMNS,
+    }
+
+
 def build_profile(df):
     rows, columns = df.shape
     numeric_cols, categorical_cols, date_cols = detect_column_types(df)
 
-    missing_counts = df.isna().sum()
+    missing_counts = _missing_counts(df)
     missing_percent = (missing_counts / max(rows, 1) * 100).round(2)
-    duplicate_rows = int(df.duplicated().sum())
+    duplicate_rows, duplicate_metadata = _duplicate_profile(df)
 
     numeric_summary = {}
     if numeric_cols:
@@ -63,17 +147,9 @@ def build_profile(df):
             "top_values": {str(k): int(v) for k, v in counts.items()},
         }
 
-    correlations = {}
-    if len(numeric_cols) >= 2:
-        correlations = (
-            df[numeric_cols]
-            .corr(numeric_only=True)
-            .round(3)
-            .replace({np.nan: None})
-            .to_dict()
-        )
+    correlations, correlation_metadata = _bounded_correlations(df, numeric_cols)
 
-    preview = df.head(15).where(df.notna(), None).to_dict(orient="records")
+    preview = df.head(15).where(df.head(15).notna(), None).to_dict(orient="records")
 
     return {
         "shape": {"rows": rows, "columns": columns},
@@ -86,9 +162,11 @@ def build_profile(df):
         "missing_percent": series_to_dict(missing_percent),
         "duplicate_rows": duplicate_rows,
         "duplicate_percent": round(duplicate_rows / max(rows, 1) * 100, 2),
+        "duplicate_metadata": duplicate_metadata,
         "memory_usage_mb": round(df.memory_usage(deep=True).sum() / (1024 * 1024), 3),
         "numeric_summary": numeric_summary,
         "categorical_summary": categorical_summary,
         "correlations": correlations,
+        "correlation_metadata": correlation_metadata,
         "preview": preview,
     }

From 9b53616743b04267d11a62f1bf8c35d9aa395fe3 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:41:48 +0530
Subject: [PATCH 106/439] perf: reuse profiler correlations for dataset signals

---
 src/framevitals/dataset_signals.py | 58 +++++++++++++++++-------------
 1 file changed, 34 insertions(+), 24 deletions(-)

diff --git a/src/framevitals/dataset_signals.py b/src/framevitals/dataset_signals.py
index 950e971..660600e 100644
--- a/src/framevitals/dataset_signals.py
+++ b/src/framevitals/dataset_signals.py
@@ -4,38 +4,45 @@
 Produces a flat dictionary of boolean + numeric signals describing the
 structural characteristics of the dataset.
 
-Signals reuse the role map when available so semantic/type scans are performed
-once per pipeline run rather than independently by downstream modules.
+Signals reuse the role map and profile when available so semantic/type scans
+and numeric correlation work are performed once per pipeline run.
 """
 
-import pandas as pd
-
 from framevitals.column_roles import (
     get_columns_with_role,
-    get_meaningful_numeric_columns,
     infer_column_roles,
 )
 
 
-def _detect_potential_leakage(df: pd.DataFrame, column_roles: dict) -> bool:
-    """Return whether a very high non-ID numeric correlation suggests leakage."""
-    meaningful = get_meaningful_numeric_columns(df, column_roles)
-    if len(meaningful) < 2:
-        return False
-    try:
-        corr = df[meaningful].corr(numeric_only=True).abs()
-        for i, a in enumerate(meaningful):
-            for b in meaningful[i + 1 :]:
-                val = corr.loc[a, b]
-                if pd.notna(val) and val >= 0.98:
-                    return True
-    except Exception:
-        pass
-    return False
+def _detect_potential_leakage(profile: dict, column_roles: dict) -> tuple[bool, bool]:
+    """Inspect cached profiler correlations for very high non-ID relationships."""
+    correlations = profile.get("correlations", {}) or {}
+    correlation_metadata = profile.get("correlation_metadata", {}) or {}
+    truncated = bool(correlation_metadata.get("truncated", False))
+
+    excluded = {"id_like", "time_like", "sequence_like", "constant"}
+    meaningful = {
+        column
+        for column, info in column_roles.items()
+        if info.get("is_numeric") and not set(info.get("roles", [])).intersection(excluded)
+    }
+
+    for left, values in correlations.items():
+        if left not in meaningful or not isinstance(values, dict):
+            continue
+        for right, value in values.items():
+            if right == left or right not in meaningful or value is None:
+                continue
+            try:
+                if abs(float(value)) >= 0.98:
+                    return True, truncated
+            except (TypeError, ValueError):
+                continue
+    return False, truncated
 
 
 def detect_dataset_signals(
-    df: pd.DataFrame,
+    df,
     profile: dict,
     column_roles: dict | None = None,
 ) -> dict:
@@ -81,7 +88,6 @@ def detect_dataset_signals(
     long_text = get_columns_with_role(column_roles, "long_text")
 
     # Semantic/value-pattern roles are already computed during role inference.
-    # Reusing them here avoids additional text-column samples/scans.
     email_like = get_columns_with_role(column_roles, "email_like")
     url_like = get_columns_with_role(column_roles, "url_like")
     uuid_like = get_columns_with_role(column_roles, "uuid_like")
@@ -92,9 +98,12 @@ def detect_dataset_signals(
     json_like = get_columns_with_role(column_roles, "json_like")
     boolean_token_like = get_columns_with_role(column_roles, "boolean_token_like")
 
-    has_leakage_risk = _detect_potential_leakage(df, column_roles)
+    has_leakage_risk, leakage_scan_truncated = _detect_potential_leakage(
+        profile,
+        column_roles,
+    )
 
-    lower_map = {c.lower(): c for c in df.columns}
+    lower_map = {str(c).lower(): c for c in df.columns}
     has_bid_ask = "bid" in lower_map and "ask" in lower_map
     has_time_series = len(time_like) > 0 and len(numeric_cols) >= 1 and rows >= 20
 
@@ -136,6 +145,7 @@ def detect_dataset_signals(
         "has_json_like_columns": len(json_like) > 0,
         "has_boolean_token_like_columns": len(boolean_token_like) > 0,
         "has_potential_leakage": has_leakage_risk,
+        "leakage_scan_truncated": leakage_scan_truncated,
         "has_target_candidates": len(target_candidates) > 0,
         "is_small_dataset": rows < 200,
         "is_large_dataset": rows >= 100000,

From 864e6dd49b337641f26f4d310198baedad2941c2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:44:25 +0530
Subject: [PATCH 107/439] fix: complete public quality gate exports

---
 src/framevitals/__init__.py | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 3cc89ed..532fe8d 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -6,7 +6,7 @@
 
 from framevitals.config import AnalysisConfig, available_modules
 from framevitals.planning import AnalysisPlan
-from framevitals.quality_results import DriftResult, ValidationResult
+from framevitals.quality_results import DriftResult, GateResult, ValidationResult
 from framevitals.result import AnalysisResult, ColumnResult
 from framevitals.snapshots import (
     AnalysisSnapshot,
@@ -228,6 +228,32 @@ def validate(data: Any, contract: dict[str, Any]) -> ValidationResult:
     return _validate(data, contract)
 
 
+def gate(
+    current: Any,
+    *,
+    reference: Any = None,
+    contract: Any = None,
+    columns: list[str] | None = None,
+    max_columns: int = 30,
+    drift_warn_on: str = "moderate",
+    drift_fail_on: str = "severe",
+    fail_on_validation_warning: bool = False,
+) -> GateResult:
+    """Run contract/drift checks and return one CI-friendly quality verdict."""
+    from framevitals.api import gate as _gate
+
+    return _gate(
+        current,
+        reference=reference,
+        contract=contract,
+        columns=columns,
+        max_columns=max_columns,
+        drift_warn_on=drift_warn_on,
+        drift_fail_on=drift_fail_on,
+        fail_on_validation_warning=fail_on_validation_warning,
+    )
+
+
 __all__ = [
     "AnalysisConfig",
     "AnalysisPlan",
@@ -236,6 +262,7 @@ def validate(data: Any, contract: dict[str, Any]) -> ValidationResult:
     "CleaningPlan",
     "ColumnResult",
     "DriftResult",
+    "GateResult",
     "ValidationResult",
     "profile",
     "roles",
@@ -252,6 +279,7 @@ def validate(data: Any, contract: dict[str, Any]) -> ValidationResult:
     "compare",
     "infer_contract",
     "validate",
+    "gate",
     "available_modules",
     "create_snapshot",
     "load_snapshot",

From a599262743cb4a4794ea189e2ce9bab3a9916b2b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:44:41 +0530
Subject: [PATCH 108/439] fix: keep ml_readiness top-level API callable

---
 src/framevitals/ml_readiness.py | 32 +++++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/ml_readiness.py b/src/framevitals/ml_readiness.py
index b32dce7..79e9db3 100644
--- a/src/framevitals/ml_readiness.py
+++ b/src/framevitals/ml_readiness.py
@@ -1,3 +1,11 @@
+"""ML-readiness scoring and compatibility helpers."""
+
+from __future__ import annotations
+
+import sys
+from types import ModuleType
+from typing import Any
+
 import numpy as np
 
 
@@ -14,7 +22,10 @@ def calculate_ml_readiness(df, profile=None):
     rows, columns = df.shape
 
     if profile is None:
-        missing_percent = float(df.isna().sum().sum() / max(rows * columns, 1) * 100)
+        # Standalone compatibility path. The public focused API builds a profile
+        # first and therefore avoids repeating these scans.
+        missing_total = sum(int(df[column].isna().sum()) for column in df.columns)
+        missing_percent = float(missing_total / max(rows * columns, 1) * 100)
         duplicate_percent = float(df.duplicated().sum() / max(rows, 1) * 100)
         categorical_cols = df.select_dtypes(include=_CATEGORICAL_DTYPES).columns.tolist()
         numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
@@ -62,3 +73,22 @@ def calculate_ml_readiness(df, profile=None):
             "Select a clear target column for supervised learning.",
         ],
     }
+
+
+class _CallableReadinessModule(ModuleType):
+    """Preserve ``fv.ml_readiness(data)`` after importing this compatibility module.
+
+    Python normally places an imported submodule on its parent package under the
+    same attribute name. Since FrameVitals historically also exposes the focused
+    function ``framevitals.ml_readiness(...)``, importing this module would
+    otherwise replace that function with a non-callable module object. Making
+    the compatibility module callable preserves both APIs without eager imports.
+    """
+
+    def __call__(self, data: Any) -> dict[str, Any]:
+        from framevitals.api import ml_readiness as public_ml_readiness
+
+        return public_ml_readiness(data)
+
+
+sys.modules[__name__].__class__ = _CallableReadinessModule

From 134d0fa15a1feac52420b349ef4c82848e56d04a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:44:58 +0530
Subject: [PATCH 109/439] test: use nondegenerate IQR anomaly fixture

---
 tests/test_scalable_foundations.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_scalable_foundations.py b/tests/test_scalable_foundations.py
index 8387a52..22e70e3 100644
--- a/tests/test_scalable_foundations.py
+++ b/tests/test_scalable_foundations.py
@@ -1,5 +1,3 @@
-from pathlib import Path
-
 import numpy as np
 import pandas as pd
 
@@ -12,9 +10,11 @@
 
 
 def test_advanced_anomaly_scores_match_expected_iqr_density():
+    # Keep IQR non-zero so the existing rule is applicable; the final row is an
+    # obvious outlier in both columns and should therefore have density 1.0.
     frame = pd.DataFrame({
-        "a": [0, 0, 0, 0, 100],
-        "b": [1, 1, 1, 1, 50],
+        "a": [0, 1, 2, 3, 100],
+        "b": [1, 2, 3, 4, 50],
     })
 
     result = calculate_anomalies(frame)

From db83d4c19e7636e43cbe6268fd66cf73f4dbff03 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:47:19 +0530
Subject: [PATCH 110/439] feat: add sparse numeric relationship graph

---
 src/framevitals/relationship_graph.py | 367 ++++++++++++++++++++++++++
 1 file changed, 367 insertions(+)
 create mode 100644 src/framevitals/relationship_graph.py

diff --git a/src/framevitals/relationship_graph.py b/src/framevitals/relationship_graph.py
new file mode 100644
index 0000000..40dd000
--- /dev/null
+++ b/src/framevitals/relationship_graph.py
@@ -0,0 +1,367 @@
+"""Sparse feature-relationship discovery for wide numeric datasets.
+
+The engine deliberately avoids allocating a dense ``columns x columns`` matrix.
+It builds compact SimHash-style signatures from a bounded row view, uses
+locality-sensitive hash bands to generate a bounded candidate set, and verifies
+only those candidates with Pearson correlation on the sampled observations.
+
+This Python implementation defines the semantics for the future Rust engine.
+The native implementation can replace the scanning/signature/candidate kernels
+without changing the result contract.
+"""
+
+from __future__ import annotations
+
+from collections import Counter, OrderedDict
+from dataclasses import dataclass
+import heapq
+import math
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+
+@dataclass(slots=True)
+class _UnionFind:
+    parent: dict[int, int]
+    size: dict[int, int]
+
+    @classmethod
+    def create(cls) -> "_UnionFind":
+        return cls(parent={}, size={})
+
+    def _ensure(self, item: int) -> None:
+        if item not in self.parent:
+            self.parent[item] = item
+            self.size[item] = 1
+
+    def find(self, item: int) -> int:
+        self._ensure(item)
+        root = item
+        while self.parent[root] != root:
+            root = self.parent[root]
+        while self.parent[item] != item:
+            parent = self.parent[item]
+            self.parent[item] = root
+            item = parent
+        return root
+
+    def union(self, left: int, right: int) -> None:
+        left_root = self.find(left)
+        right_root = self.find(right)
+        if left_root == right_root:
+            return
+        if self.size[left_root] < self.size[right_root]:
+            left_root, right_root = right_root, left_root
+        self.parent[right_root] = left_root
+        self.size[left_root] += self.size[right_root]
+
+
+def _sample_positions(rows: int, max_rows: int) -> np.ndarray:
+    if rows <= max_rows:
+        return np.arange(rows, dtype=np.int64)
+    positions = np.linspace(0, rows - 1, num=max_rows, dtype=np.int64)
+    return np.unique(positions)
+
+
+def _numeric_sample(
+    dataframe: pd.DataFrame,
+    column: Any,
+    positions: np.ndarray,
+) -> np.ndarray:
+    series = pd.to_numeric(dataframe[column].iloc[positions], errors="coerce")
+    return series.to_numpy(dtype="float64", na_value=np.nan)
+
+
+def _standardized_vector(values: np.ndarray, *, min_observations: int) -> np.ndarray | None:
+    finite = np.isfinite(values)
+    if int(finite.sum()) < min_observations:
+        return None
+
+    clean = values[finite]
+    median = float(np.median(clean))
+    work = np.where(finite, values, median).astype(np.float32, copy=False)
+    mean = float(work.mean())
+    std = float(work.std())
+    if not math.isfinite(std) or std <= 1e-12:
+        return None
+    return ((work - mean) / std).astype(np.float32, copy=False)
+
+
+def _signature(vector: np.ndarray, hyperplanes: np.ndarray) -> int:
+    bits = np.asarray(vector @ hyperplanes >= 0, dtype=np.uint8)
+    packed = np.packbits(bits, bitorder="little").tobytes()
+    return int.from_bytes(packed, byteorder="little", signed=False)
+
+
+def _absolute_band(signature: int, shift: int, band_bits: int) -> int:
+    """Canonicalize a signature band so positive/negative correlation can meet."""
+    mask = (1 << band_bits) - 1
+    value = (signature >> shift) & mask
+    complement = mask ^ value
+    return min(value, complement)
+
+
+def _pearson_from_sample(left: np.ndarray, right: np.ndarray) -> tuple[float | None, int]:
+    mask = np.isfinite(left) & np.isfinite(right)
+    overlap = int(mask.sum())
+    if overlap < 10:
+        return None, overlap
+
+    x = left[mask].astype(np.float64, copy=False)
+    y = right[mask].astype(np.float64, copy=False)
+    x = x - x.mean()
+    y = y - y.mean()
+    denominator = float(np.linalg.norm(x) * np.linalg.norm(y))
+    if denominator <= 1e-12:
+        return None, overlap
+    correlation = float(np.dot(x, y) / denominator)
+    if not math.isfinite(correlation):
+        return None, overlap
+    return max(-1.0, min(1.0, correlation)), overlap
+
+
+class _SampleCache:
+    """Tiny LRU cache so candidate verification does not retain the wide sample."""
+
+    def __init__(
+        self,
+        dataframe: pd.DataFrame,
+        columns: list[Any],
+        positions: np.ndarray,
+        *,
+        max_columns: int,
+    ) -> None:
+        self.dataframe = dataframe
+        self.columns = columns
+        self.positions = positions
+        self.max_columns = max_columns
+        self.cache: OrderedDict[int, np.ndarray] = OrderedDict()
+
+    def get(self, index: int) -> np.ndarray:
+        if index in self.cache:
+            value = self.cache.pop(index)
+            self.cache[index] = value
+            return value
+
+        value = _numeric_sample(
+            self.dataframe,
+            self.columns[index],
+            self.positions,
+        )
+        self.cache[index] = value
+        if len(self.cache) > self.max_columns:
+            self.cache.popitem(last=False)
+        return value
+
+
+def build_numeric_relationship_graph(
+    dataframe: pd.DataFrame,
+    *,
+    max_sample_rows: int = 512,
+    projections: int = 64,
+    band_bits: int = 16,
+    neighbors_per_bucket: int = 4,
+    max_bucket_members: int = 64,
+    max_candidate_pairs: int = 250_000,
+    min_abs_correlation: float = 0.80,
+    max_edges_returned: int = 5_000,
+    sample_cache_columns: int = 128,
+    random_state: int = 42,
+) -> dict[str, Any]:
+    """Discover strong numeric relationships without a dense correlation matrix.
+
+    Candidate generation is approximate; candidate verification is a normal
+    Pearson calculation on the bounded row view. The result always reports the
+    candidate/row budgets and whether either was truncated.
+    """
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError("dataframe must be a pandas DataFrame.")
+    if max_sample_rows < 20:
+        raise ValueError("max_sample_rows must be at least 20.")
+    if projections < 16 or projections > 256 or projections % 8:
+        raise ValueError("projections must be a multiple of 8 between 16 and 256.")
+    if band_bits < 4 or projections % band_bits:
+        raise ValueError("band_bits must divide projections and be at least 4.")
+    if neighbors_per_bucket < 1:
+        raise ValueError("neighbors_per_bucket must be at least 1.")
+    if max_bucket_members < neighbors_per_bucket:
+        raise ValueError("max_bucket_members must be >= neighbors_per_bucket.")
+    if max_candidate_pairs < 1:
+        raise ValueError("max_candidate_pairs must be at least 1.")
+    if not 0 < min_abs_correlation <= 1:
+        raise ValueError("min_abs_correlation must be in (0, 1].")
+    if max_edges_returned < 1:
+        raise ValueError("max_edges_returned must be at least 1.")
+    if sample_cache_columns < 2:
+        raise ValueError("sample_cache_columns must be at least 2.")
+
+    columns = dataframe.select_dtypes(include=[np.number]).columns.tolist()
+    node_count = len(columns)
+    if node_count < 2:
+        return {
+            "available": False,
+            "reason": "Need at least two numeric columns.",
+            "nodes": node_count,
+            "edges": [],
+        }
+
+    positions = _sample_positions(len(dataframe), max_sample_rows)
+    sample_rows = int(len(positions))
+    min_observations = min(20, max(10, sample_rows // 4))
+
+    rng = np.random.default_rng(random_state)
+    hyperplanes = rng.choice(
+        np.array([-1.0, 1.0], dtype=np.float32),
+        size=(sample_rows, projections),
+    )
+    hyperplanes /= math.sqrt(max(sample_rows, 1))
+
+    signatures: dict[int, int] = {}
+    skipped_columns: list[str] = []
+    for index, column in enumerate(columns):
+        raw = _numeric_sample(dataframe, column, positions)
+        vector = _standardized_vector(raw, min_observations=min_observations)
+        if vector is None:
+            skipped_columns.append(str(column))
+            continue
+        signatures[index] = _signature(vector, hyperplanes)
+
+    band_count = projections // band_bits
+    buckets: dict[tuple[int, int], list[int]] = {}
+    candidates: set[tuple[int, int]] = set()
+    candidate_truncated = False
+
+    for index in sorted(signatures):
+        signature = signatures[index]
+        for band in range(band_count):
+            key = (
+                band,
+                _absolute_band(signature, band * band_bits, band_bits),
+            )
+            members = buckets.setdefault(key, [])
+            for other in members[-neighbors_per_bucket:]:
+                pair = (other, index) if other < index else (index, other)
+                candidates.add(pair)
+                if len(candidates) >= max_candidate_pairs:
+                    candidate_truncated = True
+                    break
+            if len(members) >= max_bucket_members:
+                del members[: len(members) - max_bucket_members + 1]
+            members.append(index)
+            if candidate_truncated:
+                break
+        if candidate_truncated:
+            break
+
+    total_possible_pairs = node_count * (node_count - 1) // 2
+    cache = _SampleCache(
+        dataframe,
+        columns,
+        positions,
+        max_columns=sample_cache_columns,
+    )
+
+    union_find = _UnionFind.create()
+    degrees: Counter[int] = Counter()
+    verified_edges = 0
+    top_edge_heap: list[tuple[float, int, int, float, int]] = []
+
+    for left, right in sorted(candidates):
+        correlation, overlap = _pearson_from_sample(cache.get(left), cache.get(right))
+        if correlation is None or abs(correlation) < min_abs_correlation:
+            continue
+
+        verified_edges += 1
+        union_find.union(left, right)
+        degrees[left] += 1
+        degrees[right] += 1
+        item = (abs(correlation), left, right, correlation, overlap)
+        if len(top_edge_heap) < max_edges_returned:
+            heapq.heappush(top_edge_heap, item)
+        elif item[0] > top_edge_heap[0][0]:
+            heapq.heapreplace(top_edge_heap, item)
+
+    edges = [
+        {
+            "source": str(columns[left]),
+            "target": str(columns[right]),
+            "correlation": round(float(correlation), 6),
+            "abs_correlation": round(float(score), 6),
+            "overlap": int(overlap),
+        }
+        for score, left, right, correlation, overlap in sorted(
+            top_edge_heap,
+            key=lambda item: (-item[0], str(columns[item[1]]), str(columns[item[2]])),
+        )
+    ]
+
+    component_members: dict[int, list[int]] = {}
+    for index in union_find.parent:
+        root = union_find.find(index)
+        component_members.setdefault(root, []).append(index)
+
+    components = sorted(
+        component_members.values(),
+        key=lambda members: (-len(members), [str(columns[index]) for index in members]),
+    )
+    component_summary = [
+        {
+            "size": len(members),
+            "members": [str(columns[index]) for index in sorted(members)[:20]],
+            "members_truncated": len(members) > 20,
+        }
+        for members in components[:50]
+        if len(members) >= 2
+    ]
+
+    connected_nodes = len({index for pair in candidates for index in pair if degrees[index]})
+    high_degree = sorted(
+        (
+            {"column": str(columns[index]), "degree": int(degree)}
+            for index, degree in degrees.items()
+        ),
+        key=lambda item: (-item["degree"], item["column"]),
+    )[:25]
+
+    return {
+        "available": True,
+        "method": "bounded_simhash_lsh_then_pearson",
+        "nodes": node_count,
+        "usable_signature_nodes": len(signatures),
+        "skipped_nodes": len(skipped_columns),
+        "skipped_columns": skipped_columns[:25],
+        "sample": {
+            "source_rows": int(len(dataframe)),
+            "sample_rows": sample_rows,
+            "sampled": len(dataframe) > sample_rows,
+            "strategy": "deterministic_evenly_spaced",
+        },
+        "candidate_generation": {
+            "projections": projections,
+            "band_bits": band_bits,
+            "bands": band_count,
+            "neighbors_per_bucket": neighbors_per_bucket,
+            "candidate_pairs": len(candidates),
+            "max_candidate_pairs": max_candidate_pairs,
+            "truncated": candidate_truncated,
+            "dense_pairs_avoided": max(total_possible_pairs - len(candidates), 0),
+            "total_possible_dense_pairs": total_possible_pairs,
+        },
+        "verification": {
+            "method": "sample_pearson",
+            "min_abs_correlation": float(min_abs_correlation),
+            "verified_relationships": verified_edges,
+            "edges_returned": len(edges),
+            "edges_truncated": verified_edges > len(edges),
+        },
+        "graph": {
+            "connected_nodes": connected_nodes,
+            "isolated_or_unverified_nodes": max(node_count - connected_nodes, 0),
+            "component_count": len(component_summary),
+            "components": component_summary,
+            "high_degree_nodes": high_degree,
+        },
+        "edges": edges,
+    }

From b745ee0bf441bad00c71270fe341d5e264468103 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:47:35 +0530
Subject: [PATCH 111/439] test: cover sparse relationship discovery

---
 tests/test_relationship_graph.py | 99 ++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 tests/test_relationship_graph.py

diff --git a/tests/test_relationship_graph.py b/tests/test_relationship_graph.py
new file mode 100644
index 0000000..e1465a5
--- /dev/null
+++ b/tests/test_relationship_graph.py
@@ -0,0 +1,99 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from framevitals.relationship_graph import build_numeric_relationship_graph
+
+
+def test_relationship_graph_finds_strong_cluster_without_dense_matrix():
+    rng = np.random.default_rng(7)
+    rows = 600
+    base = rng.normal(size=rows)
+    frame = {
+        "base": base,
+        "base_copy": base.copy(),
+        "base_negative": -base,
+    }
+    for index in range(57):
+        frame[f"noise_{index}"] = rng.normal(size=rows)
+    dataframe = pd.DataFrame(frame)
+
+    result = build_numeric_relationship_graph(
+        dataframe,
+        max_sample_rows=256,
+        min_abs_correlation=0.95,
+    )
+
+    assert result["available"] is True
+    assert result["nodes"] == 60
+    assert result["method"] == "bounded_simhash_lsh_then_pearson"
+
+    pairs = {
+        frozenset((edge["source"], edge["target"]))
+        for edge in result["edges"]
+    }
+    assert frozenset(("base", "base_copy")) in pairs
+    assert frozenset(("base", "base_negative")) in pairs
+
+    candidate = result["candidate_generation"]
+    assert candidate["candidate_pairs"] < candidate["total_possible_dense_pairs"]
+    assert candidate["dense_pairs_avoided"] > 0
+
+    largest = result["graph"]["components"][0]
+    assert largest["size"] >= 3
+    assert {"base", "base_copy", "base_negative"}.issubset(largest["members"])
+
+
+def test_relationship_graph_is_row_bounded():
+    rng = np.random.default_rng(11)
+    dataframe = pd.DataFrame(
+        rng.normal(size=(5_000, 40)),
+        columns=[f"x_{index}" for index in range(40)],
+    )
+
+    result = build_numeric_relationship_graph(dataframe, max_sample_rows=128)
+
+    assert result["sample"]["source_rows"] == 5_000
+    assert result["sample"]["sample_rows"] == 128
+    assert result["sample"]["sampled"] is True
+
+
+def test_relationship_graph_candidate_budget_is_explicit():
+    rng = np.random.default_rng(3)
+    base = rng.normal(size=200)
+    dataframe = pd.DataFrame({
+        f"copy_{index}": base + rng.normal(scale=1e-8, size=200)
+        for index in range(100)
+    })
+
+    result = build_numeric_relationship_graph(
+        dataframe,
+        max_sample_rows=128,
+        max_candidate_pairs=50,
+        min_abs_correlation=0.90,
+    )
+
+    assert result["candidate_generation"]["candidate_pairs"] <= 50
+    assert result["candidate_generation"]["truncated"] is True
+    assert result["verification"]["verified_relationships"] > 0
+
+
+def test_relationship_graph_rejects_invalid_controls():
+    dataframe = pd.DataFrame({"a": range(30), "b": range(30)})
+
+    with pytest.raises(ValueError, match="max_sample_rows"):
+        build_numeric_relationship_graph(dataframe, max_sample_rows=10)
+    with pytest.raises(ValueError, match="projections"):
+        build_numeric_relationship_graph(dataframe, projections=17)
+    with pytest.raises(ValueError, match="min_abs_correlation"):
+        build_numeric_relationship_graph(dataframe, min_abs_correlation=0)
+
+
+def test_relationship_graph_handles_insufficient_numeric_columns():
+    result = build_numeric_relationship_graph(
+        pd.DataFrame({"label": ["a", "b", "c"]})
+    )
+
+    assert result["available"] is False
+    assert result["nodes"] == 0
+    assert result["edges"] == []

From ec27315d281140972d554c356006981a99d08f07 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:48:04 +0530
Subject: [PATCH 112/439] feat: expose sparse relationship graph API

---
 src/framevitals/__init__.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 532fe8d..02a94bb 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -108,6 +108,36 @@ def anomalies(
     )
 
 
+def relationships(
+    data: Any,
+    *,
+    max_sample_rows: int = 512,
+    projections: int = 64,
+    min_abs_correlation: float = 0.80,
+    max_candidate_pairs: int = 250_000,
+    max_edges_returned: int = 5_000,
+) -> dict[str, Any]:
+    """Discover strong numeric relationships without a dense correlation matrix."""
+    from framevitals.relationship_graph import build_numeric_relationship_graph
+    from framevitals.sources import resolve_source
+
+    source = resolve_source(data)
+    metadata = source.inspect()
+    dataframe = source.load()
+    payload = build_numeric_relationship_graph(
+        dataframe,
+        max_sample_rows=max_sample_rows,
+        projections=projections,
+        min_abs_correlation=min_abs_correlation,
+        max_candidate_pairs=max_candidate_pairs,
+        max_edges_returned=max_edges_returned,
+    )
+    return {
+        "dataset_name": metadata.name,
+        **payload,
+    }
+
+
 def target_analysis(data: Any, *, target: str) -> dict[str, Any]:
     """Run target quality, leakage, association, and split diagnostics only."""
     from framevitals.api import target_analysis as _target_analysis
@@ -271,6 +301,7 @@ def gate(
     "quality",
     "statistics",
     "anomalies",
+    "relationships",
     "target_analysis",
     "analyze",
     "plan",

From 0bbdbe0baeb4065de7762dddf6a9244e542cc46e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:48:46 +0530
Subject: [PATCH 113/439] test: cover public relationships API

---
 tests/test_relationship_graph.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tests/test_relationship_graph.py b/tests/test_relationship_graph.py
index e1465a5..13d27f0 100644
--- a/tests/test_relationship_graph.py
+++ b/tests/test_relationship_graph.py
@@ -2,6 +2,7 @@
 import pandas as pd
 import pytest
 
+import framevitals as fv
 from framevitals.relationship_graph import build_numeric_relationship_graph
 
 
@@ -97,3 +98,23 @@ def test_relationship_graph_handles_insufficient_numeric_columns():
     assert result["available"] is False
     assert result["nodes"] == 0
     assert result["edges"] == []
+
+
+def test_public_relationships_api_preserves_dataset_name(tmp_path):
+    path = tmp_path / "related.csv"
+    dataframe = pd.DataFrame({
+        "a": np.arange(100, dtype=float),
+        "b": np.arange(100, dtype=float) * 2,
+        "noise": np.random.default_rng(9).normal(size=100),
+    })
+    dataframe.to_csv(path, index=False)
+
+    result = fv.relationships(path, min_abs_correlation=0.95)
+
+    assert result["dataset_name"] == "related.csv"
+    assert result["available"] is True
+    pairs = {
+        frozenset((edge["source"], edge["target"]))
+        for edge in result["edges"]
+    }
+    assert frozenset(("a", "b")) in pairs

From c59f98429090b45aac0ceef2ed3f966eabec2fa4 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:49:10 +0530
Subject: [PATCH 114/439] feat: start FrameVitals Rust core workspace

---
 rust/Cargo.toml | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 rust/Cargo.toml

diff --git a/rust/Cargo.toml b/rust/Cargo.toml
new file mode 100644
index 0000000..6fbe462
--- /dev/null
+++ b/rust/Cargo.toml
@@ -0,0 +1,3 @@
+[workspace]
+members = ["framevitals-core"]
+resolver = "2"

From 4e232b6368439731c41b064e3d9b0463d9596dbe Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:49:16 +0530
Subject: [PATCH 115/439] feat: define FrameVitals native core crate

---
 rust/framevitals-core/Cargo.toml | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 rust/framevitals-core/Cargo.toml

diff --git a/rust/framevitals-core/Cargo.toml b/rust/framevitals-core/Cargo.toml
new file mode 100644
index 0000000..9a605f8
--- /dev/null
+++ b/rust/framevitals-core/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "framevitals-core"
+version = "0.1.0"
+edition = "2021"
+description = "Native streaming kernels for FrameVitals"
+license = "MIT"
+publish = false
+
+[lib]
+name = "framevitals_core"
+path = "src/lib.rs"

From 38ce93245aef2d72fb3833a3eb0aeb7f634c3a37 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:49:33 +0530
Subject: [PATCH 116/439] feat: add mergeable Rust numeric streaming state

---
 rust/framevitals-core/src/lib.rs | 225 +++++++++++++++++++++++++++++++
 1 file changed, 225 insertions(+)
 create mode 100644 rust/framevitals-core/src/lib.rs

diff --git a/rust/framevitals-core/src/lib.rs b/rust/framevitals-core/src/lib.rs
new file mode 100644
index 0000000..a4f6763
--- /dev/null
+++ b/rust/framevitals-core/src/lib.rs
@@ -0,0 +1,225 @@
+//! Native streaming primitives for FrameVitals.
+//!
+//! This crate intentionally starts with small, well-tested kernels whose
+//! semantics match the Python reference implementation. Higher-level Arrow,
+//! sketch, graph, and Python bindings can build on these primitives without
+//! coupling the analysis engine to pandas.
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct NumericState {
+    pub count: u64,
+    pub missing: u64,
+    pub infinite: u64,
+    pub mean: f64,
+    pub m2: f64,
+    pub minimum: Option<f64>,
+    pub maximum: Option<f64>,
+}
+
+impl Default for NumericState {
+    fn default() -> Self {
+        Self {
+            count: 0,
+            missing: 0,
+            infinite: 0,
+            mean: 0.0,
+            m2: 0.0,
+            minimum: None,
+            maximum: None,
+        }
+    }
+}
+
+impl NumericState {
+    /// Observe an optional numeric value using Welford's online algorithm.
+    ///
+    /// `None` and NaN are counted as missing. Positive/negative infinity are
+    /// counted separately and excluded from finite moments, matching the
+    /// current Python `NumericColumnState` semantics.
+    pub fn observe(&mut self, value: Option<f64>) {
+        let Some(value) = value else {
+            self.missing += 1;
+            return;
+        };
+
+        if value.is_nan() {
+            self.missing += 1;
+            return;
+        }
+        if !value.is_finite() {
+            self.infinite += 1;
+            return;
+        }
+
+        self.count += 1;
+        let delta = value - self.mean;
+        self.mean += delta / self.count as f64;
+        let delta2 = value - self.mean;
+        self.m2 += delta * delta2;
+
+        self.minimum = Some(match self.minimum {
+            Some(current) => current.min(value),
+            None => value,
+        });
+        self.maximum = Some(match self.maximum {
+            Some(current) => current.max(value),
+            None => value,
+        });
+    }
+
+    /// Scan a slice into one compact state without retaining observations.
+    pub fn from_values(values: &[Option<f64>]) -> Self {
+        let mut state = Self::default();
+        for &value in values {
+            state.observe(value);
+        }
+        state
+    }
+
+    /// Merge independently computed partition states using the parallel
+    /// variance formula. No raw observations are required.
+    #[must_use]
+    pub fn merge(self, other: Self) -> Self {
+        let missing = self.missing + other.missing;
+        let infinite = self.infinite + other.infinite;
+
+        if self.count == 0 {
+            return Self {
+                missing,
+                infinite,
+                ..other
+            };
+        }
+        if other.count == 0 {
+            return Self {
+                missing,
+                infinite,
+                ..self
+            };
+        }
+
+        let total = self.count + other.count;
+        let delta = other.mean - self.mean;
+        let total_f = total as f64;
+        let left_f = self.count as f64;
+        let right_f = other.count as f64;
+
+        let mean = self.mean + delta * right_f / total_f;
+        let m2 = self.m2
+            + other.m2
+            + delta * delta * left_f * right_f / total_f;
+
+        Self {
+            count: total,
+            missing,
+            infinite,
+            mean,
+            m2,
+            minimum: match (self.minimum, other.minimum) {
+                (Some(left), Some(right)) => Some(left.min(right)),
+                (left @ Some(_), None) => left,
+                (None, right @ Some(_)) => right,
+                (None, None) => None,
+            },
+            maximum: match (self.maximum, other.maximum) {
+                (Some(left), Some(right)) => Some(left.max(right)),
+                (left @ Some(_), None) => left,
+                (None, right @ Some(_)) => right,
+                (None, None) => None,
+            },
+        }
+    }
+
+    #[must_use]
+    pub fn variance(&self) -> Option<f64> {
+        if self.count < 2 {
+            None
+        } else {
+            Some(self.m2 / (self.count - 1) as f64)
+        }
+    }
+
+    #[must_use]
+    pub fn standard_deviation(&self) -> Option<f64> {
+        self.variance().map(f64::sqrt)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NumericState;
+
+    fn assert_close(left: f64, right: f64) {
+        let scale = left.abs().max(right.abs()).max(1.0);
+        assert!((left - right).abs() <= 1e-12 * scale, "{left} != {right}");
+    }
+
+    #[test]
+    fn scan_tracks_missing_infinite_and_moments() {
+        let values = [
+            Some(1.0),
+            Some(2.0),
+            None,
+            Some(f64::NAN),
+            Some(f64::INFINITY),
+            Some(4.0),
+        ];
+        let state = NumericState::from_values(&values);
+
+        assert_eq!(state.count, 3);
+        assert_eq!(state.missing, 2);
+        assert_eq!(state.infinite, 1);
+        assert_eq!(state.minimum, Some(1.0));
+        assert_eq!(state.maximum, Some(4.0));
+        assert_close(state.mean, 7.0 / 3.0);
+        assert_close(state.variance().unwrap(), 7.0 / 3.0);
+    }
+
+    #[test]
+    fn partition_merge_matches_single_pass() {
+        let values: Vec<Option<f64>> = (0..10_000)
+            .map(|value| {
+                if value % 97 == 0 {
+                    None
+                } else {
+                    Some(value as f64 * 0.25 - 100.0)
+                }
+            })
+            .collect();
+
+        let full = NumericState::from_values(&values);
+        let left = NumericState::from_values(&values[..3_333]);
+        let middle = NumericState::from_values(&values[3_333..7_777]);
+        let right = NumericState::from_values(&values[7_777..]);
+        let merged = left.merge(middle).merge(right);
+
+        assert_eq!(merged.count, full.count);
+        assert_eq!(merged.missing, full.missing);
+        assert_eq!(merged.infinite, full.infinite);
+        assert_eq!(merged.minimum, full.minimum);
+        assert_eq!(merged.maximum, full.maximum);
+        assert_close(merged.mean, full.mean);
+        assert_close(merged.m2, full.m2);
+        assert_close(merged.variance().unwrap(), full.variance().unwrap());
+    }
+
+    #[test]
+    fn merge_preserves_nonfinite_counts_for_empty_partition() {
+        let finite = NumericState::from_values(&[Some(1.0), Some(2.0)]);
+        let nonfinite = NumericState::from_values(&[None, Some(f64::NEG_INFINITY)]);
+        let merged = finite.merge(nonfinite);
+
+        assert_eq!(merged.count, 2);
+        assert_eq!(merged.missing, 1);
+        assert_eq!(merged.infinite, 1);
+        assert_eq!(merged.minimum, Some(1.0));
+        assert_eq!(merged.maximum, Some(2.0));
+    }
+
+    #[test]
+    fn constant_values_have_zero_sample_variance() {
+        let state = NumericState::from_values(&[Some(3.0), Some(3.0), Some(3.0)]);
+        assert_eq!(state.variance(), Some(0.0));
+        assert_eq!(state.standard_deviation(), Some(0.0));
+    }
+}

From 7a670548efae9ffa83aa6fd5f6e30ae90565afa9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:50:03 +0530
Subject: [PATCH 117/439] ci: test FrameVitals Rust core

---
 .github/workflows/test.yml | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 45ba443..d10afa4 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -86,6 +86,32 @@ jobs:
       - name: Run tests
         run: pytest -v
 
+  native-core:
+    name: native core (Rust)
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Configure Rust tooling
+        run: |
+          rustup default stable
+          rustup component add rustfmt clippy
+          rustc --version
+          cargo --version
+
+      - name: Check Rust formatting
+        run: cargo fmt --manifest-path rust/Cargo.toml --all -- --check
+
+      - name: Test Rust workspace
+        run: cargo test --manifest-path rust/Cargo.toml --workspace --all-targets
+
+      - name: Lint Rust workspace
+        run: cargo clippy --manifest-path rust/Cargo.toml --workspace --all-targets -- -D warnings
+
   optional-features:
     name: optional features (Python 3.12)
     runs-on: ubuntu-latest

From 8bead8fb99668bf0b2100abc8540952370116e5a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:50:46 +0530
Subject: [PATCH 118/439] feat: add optional Arrow support to Rust core

---
 rust/framevitals-core/Cargo.toml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/rust/framevitals-core/Cargo.toml b/rust/framevitals-core/Cargo.toml
index 9a605f8..332fe64 100644
--- a/rust/framevitals-core/Cargo.toml
+++ b/rust/framevitals-core/Cargo.toml
@@ -9,3 +9,11 @@ publish = false
 [lib]
 name = "framevitals_core"
 path = "src/lib.rs"
+
+[features]
+default = []
+arrow = ["dep:arrow-array", "dep:arrow-schema"]
+
+[dependencies]
+arrow-array = { version = "59.1", optional = true }
+arrow-schema = { version = "59.1", optional = true }

From 6340ed6e1b87268a9962c18cafa1a8c4528d1c83 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:51:13 +0530
Subject: [PATCH 119/439] feat: scan Arrow record batches in Rust

---
 rust/framevitals-core/src/arrow_scan.rs | 190 ++++++++++++++++++++++++
 1 file changed, 190 insertions(+)
 create mode 100644 rust/framevitals-core/src/arrow_scan.rs

diff --git a/rust/framevitals-core/src/arrow_scan.rs b/rust/framevitals-core/src/arrow_scan.rs
new file mode 100644
index 0000000..fd7f62b
--- /dev/null
+++ b/rust/framevitals-core/src/arrow_scan.rs
@@ -0,0 +1,190 @@
+//! Arrow-native batch scanning for FrameVitals.
+//!
+//! The scanner consumes Arrow `RecordBatch` values incrementally and produces
+//! mergeable numeric states. It never materializes a Python object per cell and
+//! does not require a complete dataset to reside in memory.
+
+use std::collections::{BTreeMap, BTreeSet};
+
+use arrow_array::{
+    Array, Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch,
+    UInt32Array, UInt64Array,
+};
+use arrow_schema::DataType;
+
+use crate::NumericState;
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct BatchNumericStates {
+    pub rows: u64,
+    pub states: BTreeMap<String, NumericState>,
+    pub skipped_columns: BTreeSet<String>,
+}
+
+impl BatchNumericStates {
+    /// Merge states from another Arrow batch/partition without retaining either
+    /// batch's raw observations.
+    #[must_use]
+    pub fn merge(mut self, other: Self) -> Self {
+        self.rows += other.rows;
+        self.skipped_columns.extend(other.skipped_columns);
+
+        for (name, state) in other.states {
+            self.states
+                .entry(name)
+                .and_modify(|current| *current = current.merge(state))
+                .or_insert(state);
+        }
+        self
+    }
+}
+
+macro_rules! scan_primitive {
+    ($array:expr) => {{
+        let array = $array;
+        let mut state = NumericState::default();
+        for index in 0..array.len() {
+            if array.is_null(index) {
+                state.observe(None);
+            } else {
+                state.observe(Some(array.value(index) as f64));
+            }
+        }
+        state
+    }};
+}
+
+/// Scan the supported primitive numeric columns of an Arrow `RecordBatch`.
+///
+/// Unsupported types are reported in `skipped_columns` rather than failing the
+/// complete batch, allowing future semantic/text/sketch scanners to process
+/// those columns independently.
+pub fn scan_record_batch(batch: &RecordBatch) -> BatchNumericStates {
+    let mut output = BatchNumericStates {
+        rows: batch.num_rows() as u64,
+        ..BatchNumericStates::default()
+    };
+
+    for (index, field) in batch.schema().fields().iter().enumerate() {
+        let name = field.name().clone();
+        let array = batch.column(index);
+
+        let state = match field.data_type() {
+            DataType::Float64 => array
+                .as_any()
+                .downcast_ref::<Float64Array>()
+                .map(|values| scan_primitive!(values)),
+            DataType::Float32 => array
+                .as_any()
+                .downcast_ref::<Float32Array>()
+                .map(|values| scan_primitive!(values)),
+            DataType::Int64 => array
+                .as_any()
+                .downcast_ref::<Int64Array>()
+                .map(|values| scan_primitive!(values)),
+            DataType::Int32 => array
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .map(|values| scan_primitive!(values)),
+            DataType::UInt64 => array
+                .as_any()
+                .downcast_ref::<UInt64Array>()
+                .map(|values| scan_primitive!(values)),
+            DataType::UInt32 => array
+                .as_any()
+                .downcast_ref::<UInt32Array>()
+                .map(|values| scan_primitive!(values)),
+            _ => None,
+        };
+
+        match state {
+            Some(state) => {
+                output.states.insert(name, state);
+            }
+            None => {
+                output.skipped_columns.insert(name);
+            }
+        }
+    }
+
+    output
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::{ArrayRef, Float64Array, Int64Array, RecordBatch, StringArray};
+
+    use super::scan_record_batch;
+
+    fn batch(values: Vec<Option<f64>>, integers: Vec<Option<i64>>) -> RecordBatch {
+        RecordBatch::try_from_iter(vec![
+            (
+                "value",
+                Arc::new(Float64Array::from(values)) as ArrayRef,
+            ),
+            (
+                "count",
+                Arc::new(Int64Array::from(integers)) as ArrayRef,
+            ),
+            (
+                "label",
+                Arc::new(StringArray::from(vec![Some("a"), None, Some("c")])) as ArrayRef,
+            ),
+        ])
+        .expect("valid test batch")
+    }
+
+    #[test]
+    fn scans_supported_numeric_columns_and_reports_others() {
+        let batch = batch(
+            vec![Some(1.0), None, Some(f64::INFINITY)],
+            vec![Some(10), Some(20), None],
+        );
+        let state = scan_record_batch(&batch);
+
+        assert_eq!(state.rows, 3);
+        assert_eq!(state.states.len(), 2);
+        assert!(state.skipped_columns.contains("label"));
+
+        let value = state.states.get("value").unwrap();
+        assert_eq!(value.count, 1);
+        assert_eq!(value.missing, 1);
+        assert_eq!(value.infinite, 1);
+        assert_eq!(value.mean, 1.0);
+
+        let count = state.states.get("count").unwrap();
+        assert_eq!(count.count, 2);
+        assert_eq!(count.missing, 1);
+        assert_eq!(count.mean, 15.0);
+    }
+
+    #[test]
+    fn merges_arrow_batches_into_one_dataset_state() {
+        let left = batch(
+            vec![Some(1.0), Some(2.0), None],
+            vec![Some(1), Some(2), Some(3)],
+        );
+        let right = batch(
+            vec![Some(4.0), Some(8.0), Some(16.0)],
+            vec![Some(4), Some(5), Some(6)],
+        );
+
+        let merged = scan_record_batch(&left).merge(scan_record_batch(&right));
+        let value = merged.states.get("value").unwrap();
+        let count = merged.states.get("count").unwrap();
+
+        assert_eq!(merged.rows, 6);
+        assert_eq!(value.count, 5);
+        assert_eq!(value.missing, 1);
+        assert_eq!(value.minimum, Some(1.0));
+        assert_eq!(value.maximum, Some(16.0));
+        assert!((value.mean - 6.2).abs() < 1e-12);
+
+        assert_eq!(count.count, 6);
+        assert_eq!(count.minimum, Some(1.0));
+        assert_eq!(count.maximum, Some(6.0));
+        assert!((count.mean - 3.5).abs() < 1e-12);
+    }
+}

From 6e9cb2c6ee40b1cf69b8b439c79e160bfcfafc44 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:51:30 +0530
Subject: [PATCH 120/439] feat: expose optional Arrow scanner

---
 rust/framevitals-core/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/rust/framevitals-core/src/lib.rs b/rust/framevitals-core/src/lib.rs
index a4f6763..d1d2705 100644
--- a/rust/framevitals-core/src/lib.rs
+++ b/rust/framevitals-core/src/lib.rs
@@ -5,6 +5,9 @@
 //! sketch, graph, and Python bindings can build on these primitives without
 //! coupling the analysis engine to pandas.
 
+#[cfg(feature = "arrow")]
+pub mod arrow_scan;
+
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub struct NumericState {
     pub count: u64,

From 2427316a9dedcdc4941fa510d988f95d7346ac54 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:51:47 +0530
Subject: [PATCH 121/439] ci: exercise Arrow-enabled Rust core

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d10afa4..e09b6ab 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -107,10 +107,10 @@ jobs:
         run: cargo fmt --manifest-path rust/Cargo.toml --all -- --check
 
       - name: Test Rust workspace
-        run: cargo test --manifest-path rust/Cargo.toml --workspace --all-targets
+        run: cargo test --manifest-path rust/Cargo.toml --workspace --all-targets --all-features
 
       - name: Lint Rust workspace
-        run: cargo clippy --manifest-path rust/Cargo.toml --workspace --all-targets -- -D warnings
+        run: cargo clippy --manifest-path rust/Cargo.toml --workspace --all-targets --all-features -- -D warnings
 
   optional-features:
     name: optional features (Python 3.12)

From 098731f40ccd0888679da12b6ed184ae2d224c58 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:52:19 +0530
Subject: [PATCH 122/439] feat: add hardware and acceleration capability
 detection

---
 src/framevitals/acceleration.py | 238 ++++++++++++++++++++++++++++++++
 1 file changed, 238 insertions(+)
 create mode 100644 src/framevitals/acceleration.py

diff --git a/src/framevitals/acceleration.py b/src/framevitals/acceleration.py
new file mode 100644
index 0000000..805c4e8
--- /dev/null
+++ b/src/framevitals/acceleration.py
@@ -0,0 +1,238 @@
+"""Hardware and optional acceleration discovery for FrameVitals.
+
+Discovery is intentionally read-only: importing or calling this module never
+installs packages and never makes CUDA a hard dependency. The future planner can
+consume this structured capability report when selecting per-operation backends.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from importlib.util import find_spec
+import os
+import platform
+import re
+import shutil
+import subprocess
+from typing import Any
+
+
+_CUDA_VERSION_RE = re.compile(r"CUDA Version:\s*([0-9]+)(?:\.([0-9]+))?")
+
+
+@dataclass(frozen=True, slots=True)
+class GpuDevice:
+    name: str
+    memory_total_mb: int | None = None
+    driver_version: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class SystemCapabilities:
+    platform: str
+    architecture: str
+    cpu_count: int | None
+    memory_total_bytes: int | None
+    native_module_available: bool
+    nvidia_driver_available: bool
+    cuda_compatibility: str | None
+    gpu_devices: tuple[GpuDevice, ...] = field(default_factory=tuple)
+    cupy_installed: bool = False
+    cupy_usable: bool = False
+    cupy_error: str | None = None
+    recommended_cupy_package: str | None = None
+    default_cpu_backend: str = "numpy"
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = asdict(self)
+        payload["gpu_devices"] = [asdict(device) for device in self.gpu_devices]
+        payload["eligible_backends"] = [self.default_cpu_backend]
+        if self.cupy_usable:
+            payload["eligible_backends"].append("cupy")
+        return payload
+
+
+def _total_memory_bytes() -> int | None:
+    """Best-effort physical-memory detection using only the standard library."""
+    try:
+        if os.name == "nt":
+            import ctypes
+
+            class MemoryStatusEx(ctypes.Structure):
+                _fields_ = [
+                    ("dwLength", ctypes.c_ulong),
+                    ("dwMemoryLoad", ctypes.c_ulong),
+                    ("ullTotalPhys", ctypes.c_ulonglong),
+                    ("ullAvailPhys", ctypes.c_ulonglong),
+                    ("ullTotalPageFile", ctypes.c_ulonglong),
+                    ("ullAvailPageFile", ctypes.c_ulonglong),
+                    ("ullTotalVirtual", ctypes.c_ulonglong),
+                    ("ullAvailVirtual", ctypes.c_ulonglong),
+                    ("ullAvailExtendedVirtual", ctypes.c_ulonglong),
+                ]
+
+            status = MemoryStatusEx()
+            status.dwLength = ctypes.sizeof(MemoryStatusEx)
+            if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
+                return int(status.ullTotalPhys)
+            return None
+
+        page_size = os.sysconf("SC_PAGE_SIZE")
+        physical_pages = os.sysconf("SC_PHYS_PAGES")
+        return int(page_size * physical_pages)
+    except (AttributeError, OSError, ValueError):
+        return None
+
+
+def _run_nvidia_smi(arguments: list[str]) -> str | None:
+    executable = shutil.which("nvidia-smi")
+    if executable is None:
+        return None
+    try:
+        completed = subprocess.run(
+            [executable, *arguments],
+            capture_output=True,
+            text=True,
+            timeout=3,
+            check=False,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return None
+    if completed.returncode != 0:
+        return None
+    return completed.stdout.strip()
+
+
+def _detect_nvidia() -> tuple[tuple[GpuDevice, ...], str | None]:
+    query = _run_nvidia_smi([
+        "--query-gpu=name,memory.total,driver_version",
+        "--format=csv,noheader,nounits",
+    ])
+    if not query:
+        return (), None
+
+    devices: list[GpuDevice] = []
+    for line in query.splitlines():
+        parts = [part.strip() for part in line.split(",")]
+        if not parts or not parts[0]:
+            continue
+        memory = None
+        if len(parts) >= 2:
+            try:
+                memory = int(float(parts[1]))
+            except ValueError:
+                memory = None
+        devices.append(
+            GpuDevice(
+                name=parts[0],
+                memory_total_mb=memory,
+                driver_version=parts[2] if len(parts) >= 3 and parts[2] else None,
+            )
+        )
+
+    banner = _run_nvidia_smi([])
+    cuda_compatibility = None
+    if banner:
+        match = _CUDA_VERSION_RE.search(banner)
+        if match:
+            major = match.group(1)
+            minor = match.group(2) or "0"
+            cuda_compatibility = f"{major}.{minor}"
+
+    return tuple(devices), cuda_compatibility
+
+
+def _recommend_cupy_package(
+    system: str,
+    cuda_compatibility: str | None,
+    *,
+    has_nvidia: bool,
+) -> str | None:
+    """Return the current CuPy wheel family when the environment is eligible."""
+    if not has_nvidia or system not in {"Linux", "Windows"}:
+        return None
+    if not cuda_compatibility:
+        return None
+
+    try:
+        major = int(cuda_compatibility.split(".", 1)[0])
+    except ValueError:
+        return None
+
+    if major >= 13:
+        return "cupy-cuda13x[ctk]"
+    if major == 12:
+        return "cupy-cuda12x[ctk]"
+    return None
+
+
+def _probe_cupy() -> tuple[bool, bool, str | None]:
+    installed = find_spec("cupy") is not None
+    if not installed:
+        return False, False, None
+
+    try:
+        import cupy
+
+        device_count = int(cupy.cuda.runtime.getDeviceCount())
+        return True, device_count > 0, None if device_count > 0 else "No CUDA devices found."
+    except Exception as exc:  # noqa: BLE001 - capability probe must fail soft
+        return True, False, f"{type(exc).__name__}: {exc}"
+
+
+def detect_system_capabilities(*, probe_gpu: bool = True) -> SystemCapabilities:
+    """Inspect CPU/native/CUDA capabilities without installing anything."""
+    system = platform.system() or "Unknown"
+    architecture = platform.machine() or "unknown"
+    native_available = find_spec("framevitals._native") is not None
+
+    devices: tuple[GpuDevice, ...] = ()
+    cuda_compatibility = None
+    if probe_gpu and system != "Darwin":
+        devices, cuda_compatibility = _detect_nvidia()
+
+    if probe_gpu:
+        cupy_installed, cupy_usable, cupy_error = _probe_cupy()
+    else:
+        cupy_installed = find_spec("cupy") is not None
+        cupy_usable = False
+        cupy_error = None
+
+    recommendation = _recommend_cupy_package(
+        system,
+        cuda_compatibility,
+        has_nvidia=bool(devices),
+    )
+
+    return SystemCapabilities(
+        platform=system,
+        architecture=architecture,
+        cpu_count=os.cpu_count(),
+        memory_total_bytes=_total_memory_bytes(),
+        native_module_available=native_available,
+        nvidia_driver_available=bool(devices),
+        cuda_compatibility=cuda_compatibility,
+        gpu_devices=devices,
+        cupy_installed=cupy_installed,
+        cupy_usable=cupy_usable,
+        cupy_error=cupy_error,
+        recommended_cupy_package=recommendation,
+        default_cpu_backend="rust" if native_available else "numpy",
+    )
+
+
+def system_info(*, probe_gpu: bool = True) -> dict[str, Any]:
+    """Return a JSON-safe report suitable for API/CLI/TUI diagnostics."""
+    capabilities = detect_system_capabilities(probe_gpu=probe_gpu)
+    payload = capabilities.to_dict()
+    payload["gpu_acceleration"] = {
+        "available": capabilities.cupy_usable,
+        "installable": bool(
+            capabilities.nvidia_driver_available
+            and capabilities.recommended_cupy_package
+            and not capabilities.cupy_installed
+        ),
+        "recommended_package": capabilities.recommended_cupy_package,
+        "automatic_install_performed": False,
+    }
+    return payload

From 5abce24beb149ac3963cd1ea8e6cb5ed01e82ed0 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:52:44 +0530
Subject: [PATCH 123/439] feat: expose acceleration system info

---
 src/framevitals/__init__.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 02a94bb..de9bd96 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -138,6 +138,13 @@ def relationships(
     }
 
 
+def system_info(*, probe_gpu: bool = True) -> dict[str, Any]:
+    """Inspect FrameVitals CPU/native/CUDA capabilities without installing anything."""
+    from framevitals.acceleration import system_info as _system_info
+
+    return _system_info(probe_gpu=probe_gpu)
+
+
 def target_analysis(data: Any, *, target: str) -> dict[str, Any]:
     """Run target quality, leakage, association, and split diagnostics only."""
     from framevitals.api import target_analysis as _target_analysis
@@ -302,6 +309,7 @@ def gate(
     "statistics",
     "anomalies",
     "relationships",
+    "system_info",
     "target_analysis",
     "analyze",
     "plan",

From c57f92429f8a9545567eca9f409e76247e1b6a5c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:52:57 +0530
Subject: [PATCH 124/439] test: cover optional acceleration discovery

---
 tests/test_acceleration.py | 68 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 tests/test_acceleration.py

diff --git a/tests/test_acceleration.py b/tests/test_acceleration.py
new file mode 100644
index 0000000..fe8bbeb
--- /dev/null
+++ b/tests/test_acceleration.py
@@ -0,0 +1,68 @@
+import framevitals as fv
+import framevitals.acceleration as acceleration
+
+
+def test_cupy_recommendation_is_platform_and_cuda_aware():
+    assert acceleration._recommend_cupy_package(
+        "Linux", "13.0", has_nvidia=True
+    ) == "cupy-cuda13x[ctk]"
+    assert acceleration._recommend_cupy_package(
+        "Windows", "12.8", has_nvidia=True
+    ) == "cupy-cuda12x[ctk]"
+    assert acceleration._recommend_cupy_package(
+        "Darwin", "13.0", has_nvidia=True
+    ) is None
+    assert acceleration._recommend_cupy_package(
+        "Linux", None, has_nvidia=True
+    ) is None
+    assert acceleration._recommend_cupy_package(
+        "Linux", "13.0", has_nvidia=False
+    ) is None
+
+
+def test_nvidia_smi_parser_is_best_effort(monkeypatch):
+    def fake_smi(arguments):
+        if arguments:
+            return "NVIDIA Test GPU, 24576, 999.1"
+        return "NVIDIA-SMI ... CUDA Version: 13.0"
+
+    monkeypatch.setattr(acceleration, "_run_nvidia_smi", fake_smi)
+
+    devices, cuda = acceleration._detect_nvidia()
+
+    assert cuda == "13.0"
+    assert len(devices) == 1
+    assert devices[0].name == "NVIDIA Test GPU"
+    assert devices[0].memory_total_mb == 24576
+    assert devices[0].driver_version == "999.1"
+
+
+def test_system_info_probe_can_run_without_gpu_initialization():
+    result = fv.system_info(probe_gpu=False)
+
+    assert result["platform"]
+    assert result["architecture"]
+    assert result["default_cpu_backend"] in {"numpy", "rust"}
+    assert result["gpu_acceleration"]["automatic_install_performed"] is False
+    assert isinstance(result["eligible_backends"], list)
+
+
+def test_gpu_install_is_only_recommended_not_performed(monkeypatch):
+    monkeypatch.setattr(acceleration.platform, "system", lambda: "Linux")
+    monkeypatch.setattr(
+        acceleration,
+        "_detect_nvidia",
+        lambda: ((acceleration.GpuDevice("GPU", 16384, "1"),), "13.0"),
+    )
+    monkeypatch.setattr(
+        acceleration,
+        "_probe_cupy",
+        lambda: (False, False, None),
+    )
+
+    result = acceleration.system_info(probe_gpu=True)
+
+    assert result["gpu_acceleration"]["available"] is False
+    assert result["gpu_acceleration"]["installable"] is True
+    assert result["gpu_acceleration"]["recommended_package"] == "cupy-cuda13x[ctk]"
+    assert result["gpu_acceleration"]["automatic_install_performed"] is False

From d8e69d480598dd7ba697b04aab7475b587296148 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:53:35 +0530
Subject: [PATCH 125/439] refactor: isolate lightweight focused analysis APIs

---
 src/framevitals/focused.py | 185 +++++++++++++++++++++++++++++++++++++
 1 file changed, 185 insertions(+)
 create mode 100644 src/framevitals/focused.py

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
new file mode 100644
index 0000000..df737e8
--- /dev/null
+++ b/src/framevitals/focused.py
@@ -0,0 +1,185 @@
+"""Focused FrameVitals analysis entry points.
+
+This module intentionally does not import the full analysis pipeline. Public
+calls such as ``fv.profile`` and ``fv.statistics`` should execute only the work
+the caller requested and should not load modeling/reporting orchestration as a
+side effect.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from framevitals.budgeted_analysis import (
+    run_budgeted_anomalies,
+    run_budgeted_deep_statistics,
+)
+from framevitals.column_roles import infer_column_roles, summarize_roles
+from framevitals.execution import derive_execution_budget
+from framevitals.health_score import calculate_health_score
+from framevitals.ml_readiness import calculate_ml_readiness
+from framevitals.profiler import build_profile
+from framevitals.quality_diagnostics import run_quality_diagnostics
+from framevitals.relationship_graph import build_numeric_relationship_graph
+from framevitals.sources import resolve_source
+from framevitals.target_intelligence import run_target_intelligence
+
+
+DataInput = str | Path | pd.DataFrame
+
+
+def _load(data: DataInput, *, label: str = "Dataset") -> tuple[pd.DataFrame, str]:
+    try:
+        source = resolve_source(data)
+    except (TypeError, ValueError, FileNotFoundError) as exc:
+        if label == "Dataset":
+            raise
+        raise type(exc)(str(exc).replace("Dataset", label, 1)) from exc
+
+    metadata = source.inspect()
+    dataframe = source.load()
+    return dataframe, metadata.name
+
+
+def _named(payload: dict[str, Any], source_name: str) -> dict[str, Any]:
+    return {"dataset_name": source_name, **payload}
+
+
+def profile(data: DataInput) -> dict[str, Any]:
+    dataframe, source_name = _load(data)
+    return _named(build_profile(dataframe), source_name)
+
+
+def roles(data: DataInput) -> dict[str, Any]:
+    dataframe, source_name = _load(data)
+    column_roles = infer_column_roles(dataframe)
+    return {
+        "dataset_name": source_name,
+        "columns": column_roles,
+        "summary": summarize_roles(column_roles),
+    }
+
+
+def health(data: DataInput) -> dict[str, Any]:
+    dataframe, source_name = _load(data)
+    dataset_profile = build_profile(dataframe)
+    return _named(calculate_health_score(dataframe, dataset_profile), source_name)
+
+
+def ml_readiness(data: DataInput) -> dict[str, Any]:
+    dataframe, source_name = _load(data)
+    dataset_profile = build_profile(dataframe)
+    return _named(
+        calculate_ml_readiness(dataframe, profile=dataset_profile),
+        source_name,
+    )
+
+
+def quality(
+    data: DataInput,
+    *,
+    max_sample_rows: int = 5_000,
+    max_columns: int = 100,
+    max_missingness_columns: int = 25,
+) -> dict[str, Any]:
+    dataframe, source_name = _load(data)
+    dataset_profile = build_profile(dataframe)
+    column_roles = infer_column_roles(dataframe)
+    payload = run_quality_diagnostics(
+        dataframe,
+        profile=dataset_profile,
+        column_roles=column_roles,
+        max_sample_rows=max_sample_rows,
+        max_columns=max_columns,
+        max_missingness_columns=max_missingness_columns,
+    )
+    return _named(payload, source_name)
+
+
+def statistics(
+    data: DataInput,
+    *,
+    max_pairs: int = 20,
+    mode: str = "standard",
+) -> dict[str, Any]:
+    """Run deep statistics through the same large-data budget as full analysis."""
+    dataframe, source_name = _load(data)
+    budget = derive_execution_budget(
+        len(dataframe),
+        len(dataframe.columns),
+        mode=mode,
+    )
+    payload = run_budgeted_deep_statistics(
+        dataframe,
+        budget=budget,
+        max_pairs=max_pairs,
+    )
+    return _named(payload, source_name)
+
+
+def anomalies(
+    data: DataInput,
+    *,
+    contamination: float = 0.05,
+    threshold: float = 0.6,
+    max_columns: int = 30,
+    top_k: int = 25,
+    mode: str = "standard",
+) -> dict[str, Any]:
+    """Run anomaly diagnostics with bounded covariance/neighbor work."""
+    dataframe, source_name = _load(data)
+    budget = derive_execution_budget(
+        len(dataframe),
+        len(dataframe.columns),
+        mode=mode,
+    )
+    payload = run_budgeted_anomalies(
+        dataframe,
+        budget=budget,
+        contamination=contamination,
+        threshold=threshold,
+        max_columns=max_columns,
+        top_k=top_k,
+    )
+    return _named(payload, source_name)
+
+
+def relationships(
+    data: DataInput,
+    *,
+    max_sample_rows: int = 512,
+    projections: int = 64,
+    min_abs_correlation: float = 0.80,
+    max_candidate_pairs: int = 250_000,
+    max_edges_returned: int = 5_000,
+) -> dict[str, Any]:
+    dataframe, source_name = _load(data)
+    payload = build_numeric_relationship_graph(
+        dataframe,
+        max_sample_rows=max_sample_rows,
+        projections=projections,
+        min_abs_correlation=min_abs_correlation,
+        max_candidate_pairs=max_candidate_pairs,
+        max_edges_returned=max_edges_returned,
+    )
+    return _named(payload, source_name)
+
+
+def target_analysis(
+    data: DataInput,
+    *,
+    target: str,
+) -> dict[str, Any]:
+    dataframe, source_name = _load(data)
+    if target not in dataframe.columns:
+        raise ValueError(f"Target column not found: {target}")
+    column_roles = infer_column_roles(dataframe)
+    payload = run_target_intelligence(
+        dataframe,
+        target_column=target,
+        column_roles=column_roles,
+    )
+    return _named(payload, source_name)

From b7d3c2c252e9a27708f90cd82aee7caa1004c962 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:53:53 +0530
Subject: [PATCH 126/439] perf: keep ml readiness focused API lightweight

---
 src/framevitals/ml_readiness.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/framevitals/ml_readiness.py b/src/framevitals/ml_readiness.py
index 79e9db3..36dc3ed 100644
--- a/src/framevitals/ml_readiness.py
+++ b/src/framevitals/ml_readiness.py
@@ -86,7 +86,7 @@ class _CallableReadinessModule(ModuleType):
     """
 
     def __call__(self, data: Any) -> dict[str, Any]:
-        from framevitals.api import ml_readiness as public_ml_readiness
+        from framevitals.focused import ml_readiness as public_ml_readiness
 
         return public_ml_readiness(data)
 

From 8b5c8ce233237b50a7a5e5362d7f97cc395798c6 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:54:20 +0530
Subject: [PATCH 127/439] perf: route focused APIs without loading full
 pipeline

---
 src/framevitals/__init__.py | 47 ++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index de9bd96..9fee038 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -37,28 +37,28 @@ def __getattr__(name: str):
 
 def profile(data: Any) -> dict[str, Any]:
     """Profile shape, dtypes, missingness, cardinality, and basic summaries."""
-    from framevitals.api import profile as _profile
+    from framevitals.focused import profile as _profile
 
     return _profile(data)
 
 
 def roles(data: Any) -> dict[str, Any]:
     """Infer semantic and structural roles for dataset columns."""
-    from framevitals.api import roles as _roles
+    from framevitals.focused import roles as _roles
 
     return _roles(data)
 
 
 def health(data: Any) -> dict[str, Any]:
     """Calculate only the FrameVitals data-health score."""
-    from framevitals.api import health as _health
+    from framevitals.focused import health as _health
 
     return _health(data)
 
 
 def ml_readiness(data: Any) -> dict[str, Any]:
     """Calculate only ML-readiness diagnostics."""
-    from framevitals.api import ml_readiness as _ml_readiness
+    from framevitals.focused import ml_readiness as _ml_readiness
 
     return _ml_readiness(data)
 
@@ -71,7 +71,7 @@ def quality(
     max_missingness_columns: int = 25,
 ) -> dict[str, Any]:
     """Run practical deterministic data-quality diagnostics only."""
-    from framevitals.api import quality as _quality
+    from framevitals.focused import quality as _quality
 
     return _quality(
         data,
@@ -81,11 +81,16 @@ def quality(
     )
 
 
-def statistics(data: Any, *, max_pairs: int = 20) -> dict[str, Any]:
-    """Run the deep statistical diagnostics layer only."""
-    from framevitals.api import statistics as _statistics
+def statistics(
+    data: Any,
+    *,
+    max_pairs: int = 20,
+    mode: str = "standard",
+) -> dict[str, Any]:
+    """Run the deep statistical diagnostics layer with adaptive row budgets."""
+    from framevitals.focused import statistics as _statistics
 
-    return _statistics(data, max_pairs=max_pairs)
+    return _statistics(data, max_pairs=max_pairs, mode=mode)
 
 
 def anomalies(
@@ -95,9 +100,10 @@ def anomalies(
     threshold: float = 0.6,
     max_columns: int = 30,
     top_k: int = 25,
+    mode: str = "standard",
 ) -> dict[str, Any]:
-    """Run only the FrameVitals tabular anomaly ensemble."""
-    from framevitals.api import anomalies as _anomalies
+    """Run only the bounded FrameVitals tabular anomaly ensemble."""
+    from framevitals.focused import anomalies as _anomalies
 
     return _anomalies(
         data,
@@ -105,6 +111,7 @@ def anomalies(
         threshold=threshold,
         max_columns=max_columns,
         top_k=top_k,
+        mode=mode,
     )
 
 
@@ -118,24 +125,16 @@ def relationships(
     max_edges_returned: int = 5_000,
 ) -> dict[str, Any]:
     """Discover strong numeric relationships without a dense correlation matrix."""
-    from framevitals.relationship_graph import build_numeric_relationship_graph
-    from framevitals.sources import resolve_source
-
-    source = resolve_source(data)
-    metadata = source.inspect()
-    dataframe = source.load()
-    payload = build_numeric_relationship_graph(
-        dataframe,
+    from framevitals.focused import relationships as _relationships
+
+    return _relationships(
+        data,
         max_sample_rows=max_sample_rows,
         projections=projections,
         min_abs_correlation=min_abs_correlation,
         max_candidate_pairs=max_candidate_pairs,
         max_edges_returned=max_edges_returned,
     )
-    return {
-        "dataset_name": metadata.name,
-        **payload,
-    }
 
 
 def system_info(*, probe_gpu: bool = True) -> dict[str, Any]:
@@ -147,7 +146,7 @@ def system_info(*, probe_gpu: bool = True) -> dict[str, Any]:
 
 def target_analysis(data: Any, *, target: str) -> dict[str, Any]:
     """Run target quality, leakage, association, and split diagnostics only."""
-    from framevitals.api import target_analysis as _target_analysis
+    from framevitals.focused import target_analysis as _target_analysis
 
     return _target_analysis(data, target=target)
 

From fc9db2cf086bf6ee6676658bab6fb54f374e451b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:54:48 +0530
Subject: [PATCH 128/439] perf: lazy-load focused analysis implementations

---
 src/framevitals/focused.py | 43 +++++++++++++++++++++++++-------------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index df737e8..a42c9ce 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -1,9 +1,9 @@
 """Focused FrameVitals analysis entry points.
 
 This module intentionally does not import the full analysis pipeline. Public
-calls such as ``fv.profile`` and ``fv.statistics`` should execute only the work
-the caller requested and should not load modeling/reporting orchestration as a
-side effect.
+calls such as ``fv.profile`` and ``fv.statistics`` execute only the work the
+caller requested. Individual analysis implementations are imported lazily too,
+so a profile-only call does not load sklearn/statsmodels/deep-stat modules.
 """
 
 from __future__ import annotations
@@ -13,19 +13,7 @@
 
 import pandas as pd
 
-from framevitals.budgeted_analysis import (
-    run_budgeted_anomalies,
-    run_budgeted_deep_statistics,
-)
-from framevitals.column_roles import infer_column_roles, summarize_roles
-from framevitals.execution import derive_execution_budget
-from framevitals.health_score import calculate_health_score
-from framevitals.ml_readiness import calculate_ml_readiness
-from framevitals.profiler import build_profile
-from framevitals.quality_diagnostics import run_quality_diagnostics
-from framevitals.relationship_graph import build_numeric_relationship_graph
 from framevitals.sources import resolve_source
-from framevitals.target_intelligence import run_target_intelligence
 
 
 DataInput = str | Path | pd.DataFrame
@@ -49,11 +37,15 @@ def _named(payload: dict[str, Any], source_name: str) -> dict[str, Any]:
 
 
 def profile(data: DataInput) -> dict[str, Any]:
+    from framevitals.profiler import build_profile
+
     dataframe, source_name = _load(data)
     return _named(build_profile(dataframe), source_name)
 
 
 def roles(data: DataInput) -> dict[str, Any]:
+    from framevitals.column_roles import infer_column_roles, summarize_roles
+
     dataframe, source_name = _load(data)
     column_roles = infer_column_roles(dataframe)
     return {
@@ -64,12 +56,18 @@ def roles(data: DataInput) -> dict[str, Any]:
 
 
 def health(data: DataInput) -> dict[str, Any]:
+    from framevitals.health_score import calculate_health_score
+    from framevitals.profiler import build_profile
+
     dataframe, source_name = _load(data)
     dataset_profile = build_profile(dataframe)
     return _named(calculate_health_score(dataframe, dataset_profile), source_name)
 
 
 def ml_readiness(data: DataInput) -> dict[str, Any]:
+    from framevitals.ml_readiness import calculate_ml_readiness
+    from framevitals.profiler import build_profile
+
     dataframe, source_name = _load(data)
     dataset_profile = build_profile(dataframe)
     return _named(
@@ -85,6 +83,10 @@ def quality(
     max_columns: int = 100,
     max_missingness_columns: int = 25,
 ) -> dict[str, Any]:
+    from framevitals.column_roles import infer_column_roles
+    from framevitals.profiler import build_profile
+    from framevitals.quality_diagnostics import run_quality_diagnostics
+
     dataframe, source_name = _load(data)
     dataset_profile = build_profile(dataframe)
     column_roles = infer_column_roles(dataframe)
@@ -106,6 +108,9 @@ def statistics(
     mode: str = "standard",
 ) -> dict[str, Any]:
     """Run deep statistics through the same large-data budget as full analysis."""
+    from framevitals.budgeted_analysis import run_budgeted_deep_statistics
+    from framevitals.execution import derive_execution_budget
+
     dataframe, source_name = _load(data)
     budget = derive_execution_budget(
         len(dataframe),
@@ -130,6 +135,9 @@ def anomalies(
     mode: str = "standard",
 ) -> dict[str, Any]:
     """Run anomaly diagnostics with bounded covariance/neighbor work."""
+    from framevitals.budgeted_analysis import run_budgeted_anomalies
+    from framevitals.execution import derive_execution_budget
+
     dataframe, source_name = _load(data)
     budget = derive_execution_budget(
         len(dataframe),
@@ -156,6 +164,8 @@ def relationships(
     max_candidate_pairs: int = 250_000,
     max_edges_returned: int = 5_000,
 ) -> dict[str, Any]:
+    from framevitals.relationship_graph import build_numeric_relationship_graph
+
     dataframe, source_name = _load(data)
     payload = build_numeric_relationship_graph(
         dataframe,
@@ -173,6 +183,9 @@ def target_analysis(
     *,
     target: str,
 ) -> dict[str, Any]:
+    from framevitals.column_roles import infer_column_roles
+    from framevitals.target_intelligence import run_target_intelligence
+
     dataframe, source_name = _load(data)
     if target not in dataframe.columns:
         raise ValueError(f"Target column not found: {target}")

From a73a89644ed4335db1c0631f9e0e2945ddc1960d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:55:08 +0530
Subject: [PATCH 129/439] test: enforce focused API import boundaries

---
 tests/test_focused_import_boundaries.py | 31 +++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 tests/test_focused_import_boundaries.py

diff --git a/tests/test_focused_import_boundaries.py b/tests/test_focused_import_boundaries.py
new file mode 100644
index 0000000..9f3f086
--- /dev/null
+++ b/tests/test_focused_import_boundaries.py
@@ -0,0 +1,31 @@
+import subprocess
+import sys
+
+
+def test_profile_does_not_load_heavy_analysis_stack():
+    code = r'''
+import sys
+import pandas as pd
+import framevitals as fv
+
+df = pd.DataFrame({"x": [1, 2, 3], "label": ["a", "b", "a"]})
+result = fv.profile(df)
+assert result["shape"]["rows"] == 3
+
+for name in (
+    "framevitals.pipeline",
+    "framevitals.anomaly_ensemble",
+    "framevitals.deep_statistics_v2",
+    "sklearn",
+    "statsmodels",
+):
+    assert name not in sys.modules, (name, sorted(k for k in sys.modules if k.startswith(name)))
+'''
+    completed = subprocess.run(
+        [sys.executable, "-c", code],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+    assert completed.returncode == 0, completed.stderr

From 1fd253020727f6dde5c19b5de9b44617f28ba6a6 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:57:55 +0530
Subject: [PATCH 130/439] style: apply rustfmt to Arrow scanner

---
 rust/framevitals-core/src/arrow_scan.rs | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/rust/framevitals-core/src/arrow_scan.rs b/rust/framevitals-core/src/arrow_scan.rs
index fd7f62b..2460068 100644
--- a/rust/framevitals-core/src/arrow_scan.rs
+++ b/rust/framevitals-core/src/arrow_scan.rs
@@ -7,8 +7,8 @@
 use std::collections::{BTreeMap, BTreeSet};
 
 use arrow_array::{
-    Array, Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch,
-    UInt32Array, UInt64Array,
+    Array, Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, UInt32Array,
+    UInt64Array,
 };
 use arrow_schema::DataType;
 
@@ -120,14 +120,8 @@ mod tests {
 
     fn batch(values: Vec<Option<f64>>, integers: Vec<Option<i64>>) -> RecordBatch {
         RecordBatch::try_from_iter(vec![
-            (
-                "value",
-                Arc::new(Float64Array::from(values)) as ArrayRef,
-            ),
-            (
-                "count",
-                Arc::new(Int64Array::from(integers)) as ArrayRef,
-            ),
+            ("value", Arc::new(Float64Array::from(values)) as ArrayRef),
+            ("count", Arc::new(Int64Array::from(integers)) as ArrayRef),
             (
                 "label",
                 Arc::new(StringArray::from(vec![Some("a"), None, Some("c")])) as ArrayRef,

From 5f2970a63d002fb9c5b4ed464d4358dbba17726b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:58:19 +0530
Subject: [PATCH 131/439] style: apply rustfmt to native core

---
 rust/framevitals-core/src/lib.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/rust/framevitals-core/src/lib.rs b/rust/framevitals-core/src/lib.rs
index d1d2705..9f85051 100644
--- a/rust/framevitals-core/src/lib.rs
+++ b/rust/framevitals-core/src/lib.rs
@@ -108,9 +108,7 @@ impl NumericState {
         let right_f = other.count as f64;
 
         let mean = self.mean + delta * right_f / total_f;
-        let m2 = self.m2
-            + other.m2
-            + delta * delta * left_f * right_f / total_f;
+        let m2 = self.m2 + other.m2 + delta * delta * left_f * right_f / total_f;
 
         Self {
             count: total,

From ce4957c8db7c4620bad3b78d6423ac6813e87943 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:58:58 +0530
Subject: [PATCH 132/439] refactor: isolate budget-aware planning API

---
 src/framevitals/planning_api.py | 99 +++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 src/framevitals/planning_api.py

diff --git a/src/framevitals/planning_api.py b/src/framevitals/planning_api.py
new file mode 100644
index 0000000..da69621
--- /dev/null
+++ b/src/framevitals/planning_api.py
@@ -0,0 +1,99 @@
+"""Planning-only public execution path.
+
+`fv.plan()` should be cheap enough to call before committing to a full analysis.
+This module deliberately avoids importing the full FrameVitals pipeline while
+still resolving configuration, structural signals, and adaptive execution
+budgets exactly as the execution layer expects.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from framevitals.analysis_selector import select_analyses
+from framevitals.column_roles import infer_column_roles
+from framevitals.config import ConfigInput, VALID_MODULES, resolve_config
+from framevitals.dataset_signals import detect_dataset_signals
+from framevitals.execution import derive_execution_budget
+from framevitals.planning import AnalysisPlan
+from framevitals.profiler import build_profile
+from framevitals.sources import resolve_source
+
+
+DataInput = str | Path | pd.DataFrame
+
+
+def plan(
+    data: DataInput,
+    *,
+    target: str | None = None,
+    mode: str | None = None,
+    workers: int | None = None,
+    preset: str | None = None,
+    config: ConfigInput = None,
+    disabled_modules: list[str] | tuple[str, ...] | None = None,
+) -> AnalysisPlan:
+    """Preview planned analyses, scale policy, and execution constraints."""
+    resolved = resolve_config(
+        config,
+        preset=preset,
+        mode=mode,
+        target=target,
+        workers=workers,
+        artifacts=False,
+        disabled_modules=disabled_modules,
+    )
+
+    source = resolve_source(data)
+    source_metadata = source.inspect()
+    dataframe = source.load()
+
+    if resolved.target is not None and resolved.target not in dataframe.columns:
+        raise ValueError(f"Target column not found: {resolved.target}")
+
+    dataset_profile = build_profile(dataframe)
+    column_roles = infer_column_roles(dataframe)
+    dataset_signals = detect_dataset_signals(
+        dataframe,
+        dataset_profile,
+        column_roles=column_roles,
+    )
+
+    budget = derive_execution_budget(
+        len(dataframe),
+        len(dataframe.columns),
+        mode=resolved.mode,
+    )
+
+    selection = select_analyses(
+        signals=dataset_signals,
+        analysis_mode=resolved.mode,
+        target_column=resolved.target,
+    )
+    disabled = set(resolved.disabled_modules)
+    selection["execution_modules"] = {
+        "disabled": sorted(disabled),
+        "enabled": sorted(VALID_MODULES - disabled),
+    }
+    selection["execution_budget"] = budget.to_dict()
+
+    public_signals = {
+        key: value
+        for key, value in dataset_signals.items()
+        if key != "column_roles"
+    }
+
+    return AnalysisPlan({
+        "dataset_name": source_metadata.name,
+        "source": source_metadata.to_dict(),
+        "analysis_mode": resolved.mode,
+        "target": resolved.target,
+        "shape": dict(dataset_profile.get("shape", {})),
+        "config": resolved.to_dict(),
+        "execution_budget": budget.to_dict(),
+        "signals": public_signals,
+        "selection": selection,
+    })

From a10046b99bf4df6e02d79817c9a2ca509bb8da01 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:59:30 +0530
Subject: [PATCH 133/439] perf: keep planning off full pipeline import

---
 src/framevitals/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 9fee038..cf3820a 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -187,8 +187,8 @@ def plan(
     config: Any = None,
     disabled_modules: list[str] | tuple[str, ...] | None = None,
 ) -> AnalysisPlan:
-    """Preview applicable analyses without running heavy analysis stages."""
-    from framevitals.api import plan as _plan
+    """Preview analyses, scale policy, and execution constraints without running them."""
+    from framevitals.planning_api import plan as _plan
 
     return _plan(
         data,

From 12e704fdc40b32be6b661fbcf470a1016901ed5b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 11:59:42 +0530
Subject: [PATCH 134/439] test: cover lightweight budget-aware planning

---
 tests/test_planning_boundaries.py | 54 +++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 tests/test_planning_boundaries.py

diff --git a/tests/test_planning_boundaries.py b/tests/test_planning_boundaries.py
new file mode 100644
index 0000000..79f3ab5
--- /dev/null
+++ b/tests/test_planning_boundaries.py
@@ -0,0 +1,54 @@
+import subprocess
+import sys
+
+import pandas as pd
+
+import framevitals as fv
+
+
+def test_plan_exposes_execution_budget():
+    frame = pd.DataFrame({
+        "x": range(100),
+        "y": range(100),
+    })
+
+    result = fv.plan(frame, mode="standard")
+
+    assert result["execution_budget"]["rows"] == 100
+    assert result["execution_budget"]["columns"] == 2
+    assert result["selection"]["execution_budget"] == result["execution_budget"]
+    assert result["source"]["format"] == "pandas"
+
+
+def test_plan_classifies_extreme_shape_policy_without_changing_api():
+    # The execution policy itself can reason about extreme shape without ever
+    # allocating such a dataset. This is validated in adaptive-execution tests;
+    # here we ensure ordinary plans expose the same policy schema.
+    result = fv.plan(pd.DataFrame({"x": [1, 2, 3]}), mode="quick")
+
+    budget = result["execution_budget"]
+    assert budget["scale_class"] in {"normal", "large", "very_large", "extreme"}
+    assert "max_memory_heavy_parallelism" in budget
+    assert "bootstrap_sample_rows" in budget
+
+
+def test_plan_does_not_import_full_execution_pipeline():
+    code = r'''
+import sys
+import pandas as pd
+import framevitals as fv
+
+result = fv.plan(pd.DataFrame({"x": range(30), "y": range(30)}), mode="quick")
+assert result["analysis_mode"] == "quick"
+assert "framevitals.pipeline" not in sys.modules
+assert "framevitals.anomaly_ensemble" not in sys.modules
+assert "framevitals.deep_statistics_v2" not in sys.modules
+'''
+    completed = subprocess.run(
+        [sys.executable, "-c", code],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+    assert completed.returncode == 0, completed.stderr

From 3097ff2f5db022eac57a06a9805d3fe7db0c1479 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:00:17 +0530
Subject: [PATCH 135/439] refactor: isolate lightweight data operations

---
 src/framevitals/operations.py | 253 ++++++++++++++++++++++++++++++++++
 1 file changed, 253 insertions(+)
 create mode 100644 src/framevitals/operations.py

diff --git a/src/framevitals/operations.py b/src/framevitals/operations.py
new file mode 100644
index 0000000..1b3887c
--- /dev/null
+++ b/src/framevitals/operations.py
@@ -0,0 +1,253 @@
+"""Lightweight public data operations that do not require the full pipeline.
+
+Cleaning, contracts, drift, and quality gates are useful independently of EDA,
+modeling, explainability, and report generation. Keeping them here prevents a
+simple validation or comparison call from importing the heavy analysis stack.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from framevitals.cleaning_plan import (
+    CleaningPlan,
+    apply_cleaning_plan,
+    infer_cleaning_plan,
+)
+from framevitals.contracts import infer_contract as _infer_contract
+from framevitals.contracts import validate_contract
+from framevitals.drift_analysis import compare_datasets, severity_at_least
+from framevitals.loader import load_dataset
+from framevitals.profiler import build_profile
+from framevitals.quality_results import DriftResult, GateResult, ValidationResult
+
+
+DataInput = str | Path | pd.DataFrame
+_DRIFT_SEVERITY_RANK = {"stable": 0, "minor": 1, "moderate": 2, "severe": 3}
+
+
+def _validated_path(value: str | Path, *, label: str) -> Path:
+    path = Path(value)
+    if not path.exists():
+        raise FileNotFoundError(f"{label} not found: {path}")
+    if not path.is_file():
+        raise ValueError(f"Expected a file for {label.lower()}, got: {path}")
+    return path
+
+
+def _load_input(value: DataInput, *, label: str) -> tuple[pd.DataFrame, str]:
+    if isinstance(value, pd.DataFrame):
+        if value.empty:
+            raise ValueError(f"{label} DataFrame is empty.")
+        return value.copy(), "<dataframe>"
+
+    if isinstance(value, (str, Path)):
+        path = _validated_path(value, label=label)
+        dataframe = load_dataset(path)
+        if dataframe.empty:
+            raise ValueError(f"{label} dataset is empty: {path}")
+        return dataframe, path.name
+
+    raise TypeError(
+        f"{label} must be a pandas DataFrame or a path to a supported dataset."
+    )
+
+
+def plan_cleaning(data: DataInput) -> CleaningPlan:
+    """Infer a conservative cleaning plan without modifying the input data."""
+    dataframe, _ = _load_input(data, label="Dataset")
+    dataset_profile = build_profile(dataframe)
+    return infer_cleaning_plan(dataframe, profile=dataset_profile)
+
+
+def clean(
+    data: DataInput,
+    *,
+    plan: Mapping[str, Any] | None = None,
+) -> pd.DataFrame:
+    """Return an explicitly cleaned copy; never mutate the caller's input."""
+    dataframe, _ = _load_input(data, label="Dataset")
+    resolved_plan = plan if plan is not None else infer_cleaning_plan(dataframe)
+    return apply_cleaning_plan(dataframe, resolved_plan, copy=True)
+
+
+def compare(
+    reference: DataInput,
+    current: DataInput,
+    *,
+    columns: list[str] | None = None,
+    max_columns: int = 30,
+) -> DriftResult:
+    """Compare reference/current datasets without executing the EDA pipeline."""
+    if max_columns < 1:
+        raise ValueError("max_columns must be at least 1.")
+
+    reference_df, reference_name = _load_input(reference, label="Reference")
+    current_df, current_name = _load_input(current, label="Current")
+    payload = compare_datasets(
+        reference_df,
+        current_df,
+        columns=columns,
+        max_columns=max_columns,
+    )
+    payload["reference_name"] = reference_name
+    payload["current_name"] = current_name
+    return DriftResult(payload)
+
+
+def infer_contract(
+    data: DataInput,
+    *,
+    numeric_tolerance: float = 0.05,
+    max_categories: int = 20,
+    null_fraction_tolerance: float = 0.05,
+    infer_unique: bool = True,
+    min_unique_rows: int = 20,
+    allow_extra_columns: bool = False,
+) -> dict[str, Any]:
+    """Infer a JSON-serializable contract from a reference dataset."""
+    dataframe, source_name = _load_input(data, label="Reference")
+    contract = _infer_contract(
+        dataframe,
+        numeric_tolerance=numeric_tolerance,
+        max_categories=max_categories,
+        null_fraction_tolerance=null_fraction_tolerance,
+        infer_unique=infer_unique,
+        min_unique_rows=min_unique_rows,
+        allow_extra_columns=allow_extra_columns,
+    )
+    contract["reference_name"] = source_name
+    return contract
+
+
+def validate(
+    data: DataInput,
+    contract: Mapping[str, Any],
+) -> ValidationResult:
+    """Validate a dataset against an inferred or explicit contract."""
+    dataframe, source_name = _load_input(data, label="Dataset")
+    payload = validate_contract(dataframe, contract)
+    payload["dataset_name"] = source_name
+    return ValidationResult(payload)
+
+
+def gate(
+    current: DataInput,
+    *,
+    reference: DataInput | None = None,
+    contract: Mapping[str, Any] | None = None,
+    columns: list[str] | None = None,
+    max_columns: int = 30,
+    drift_warn_on: str = "moderate",
+    drift_fail_on: str = "severe",
+    fail_on_validation_warning: bool = False,
+) -> GateResult:
+    """Combine contract/drift checks into one CI-friendly quality verdict."""
+    if reference is None and contract is None:
+        raise ValueError("gate requires at least one of reference= or contract=.")
+    if drift_warn_on not in _DRIFT_SEVERITY_RANK:
+        raise ValueError("drift_warn_on must be one of: stable, minor, moderate, severe.")
+    if drift_fail_on not in _DRIFT_SEVERITY_RANK:
+        raise ValueError("drift_fail_on must be one of: stable, minor, moderate, severe.")
+    if _DRIFT_SEVERITY_RANK[drift_warn_on] > _DRIFT_SEVERITY_RANK[drift_fail_on]:
+        raise ValueError("drift_warn_on cannot be more severe than drift_fail_on.")
+    if max_columns < 1:
+        raise ValueError("max_columns must be at least 1.")
+
+    current_df, current_name = _load_input(current, label="Current")
+    checks: dict[str, Any] = {}
+    reasons: list[str] = []
+    status = "pass"
+
+    def warn() -> None:
+        nonlocal status
+        if status == "pass":
+            status = "warn"
+
+    def fail() -> None:
+        nonlocal status
+        status = "fail"
+
+    if contract is not None:
+        validation_payload = validate_contract(current_df, contract)
+        validation_payload["dataset_name"] = current_name
+        checks["validation"] = validation_payload
+
+        validation_status = validation_payload.get("status")
+        if validation_status == "fail":
+            fail()
+            error_count = validation_payload.get("summary", {}).get("errors", 0)
+            reasons.append(f"Contract validation failed with {error_count} error(s).")
+        elif validation_status == "warn":
+            warning_count = validation_payload.get("summary", {}).get("warnings", 0)
+            if fail_on_validation_warning:
+                fail()
+                reasons.append(
+                    f"Contract validation produced {warning_count} warning(s), promoted to failure."
+                )
+            else:
+                warn()
+                reasons.append(f"Contract validation produced {warning_count} warning(s).")
+
+    if reference is not None:
+        reference_df, reference_name = _load_input(reference, label="Reference")
+        drift_payload = compare_datasets(
+            reference_df,
+            current_df,
+            columns=columns,
+            max_columns=max_columns,
+        )
+        drift_payload["reference_name"] = reference_name
+        drift_payload["current_name"] = current_name
+        checks["drift"] = drift_payload
+
+        if not drift_payload.get("available"):
+            fail()
+            reasons.append(
+                "Drift comparison was requested but could not produce a comparable result: "
+                f"{drift_payload.get('reason', 'unknown reason')}."
+            )
+        else:
+            drift_severity = str(
+                drift_payload.get("gate", {}).get("severity", "unknown")
+            )
+            if drift_severity not in _DRIFT_SEVERITY_RANK:
+                fail()
+                reasons.append(
+                    "Drift comparison did not produce a recognized severity verdict."
+                )
+            elif severity_at_least(drift_severity, drift_fail_on):
+                fail()
+                reasons.append(
+                    f"Drift severity {drift_severity} reached fail threshold {drift_fail_on}."
+                )
+            elif severity_at_least(drift_severity, drift_warn_on):
+                warn()
+                reasons.append(
+                    f"Drift severity {drift_severity} reached warning threshold {drift_warn_on}."
+                )
+
+            drift_reasons = drift_payload.get("gate", {}).get("reasons", [])
+            if isinstance(drift_reasons, list):
+                for reason in drift_reasons[:10]:
+                    text = str(reason)
+                    if text and text not in reasons:
+                        reasons.append(text)
+
+    return GateResult({
+        "status": status,
+        "passed": status != "fail",
+        "current_name": current_name,
+        "checks_run": list(checks),
+        "thresholds": {
+            "drift_warn_on": drift_warn_on,
+            "drift_fail_on": drift_fail_on,
+            "fail_on_validation_warning": bool(fail_on_validation_warning),
+        },
+        "reasons": reasons,
+        "checks": checks,
+    })

From 84a840a8479da67e68e72beed367eaee11250be8 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:00:44 +0530
Subject: [PATCH 136/439] perf: keep data operations off full pipeline import

---
 src/framevitals/__init__.py | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index cf3820a..8cd90fc 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -22,12 +22,7 @@
 
 
 def __getattr__(name: str):
-    """Lazily expose optional/heavier public types.
-
-    ``CleaningPlan`` depends on pandas/NumPy. Delaying that import preserves the
-    package's lightweight top-level import boundary and built-wheel smoke test
-    while keeping ``framevitals.CleaningPlan`` available to normal users.
-    """
+    """Lazily expose optional/heavier public types."""
     if name == "CleaningPlan":
         from framevitals.cleaning_plan import CleaningPlan
 
@@ -203,14 +198,14 @@ def plan(
 
 def plan_cleaning(data: Any) -> CleaningPlan:
     """Infer a conservative cleaning plan without changing the input data."""
-    from framevitals.api import plan_cleaning as _plan_cleaning
+    from framevitals.operations import plan_cleaning as _plan_cleaning
 
     return _plan_cleaning(data)
 
 
 def clean(data: Any, *, plan: Any = None):
     """Return an explicitly cleaned copy using a supplied or inferred plan."""
-    from framevitals.api import clean as _clean
+    from framevitals.operations import clean as _clean
 
     return _clean(data, plan=plan)
 
@@ -223,7 +218,7 @@ def compare(
     max_columns: int = 30,
 ) -> DriftResult:
     """Compare two datasets and return a structured drift report."""
-    from framevitals.api import compare as _compare
+    from framevitals.operations import compare as _compare
 
     return _compare(
         reference,
@@ -244,7 +239,7 @@ def infer_contract(
     allow_extra_columns: bool = False,
 ) -> dict[str, Any]:
     """Infer a reusable data contract from a reference dataset."""
-    from framevitals.api import infer_contract as _infer_contract
+    from framevitals.operations import infer_contract as _infer_contract
 
     return _infer_contract(
         data,
@@ -259,7 +254,7 @@ def infer_contract(
 
 def validate(data: Any, contract: dict[str, Any]) -> ValidationResult:
     """Validate a dataset against an inferred or explicit data contract."""
-    from framevitals.api import validate as _validate
+    from framevitals.operations import validate as _validate
 
     return _validate(data, contract)
 
@@ -276,7 +271,7 @@ def gate(
     fail_on_validation_warning: bool = False,
 ) -> GateResult:
     """Run contract/drift checks and return one CI-friendly quality verdict."""
-    from framevitals.api import gate as _gate
+    from framevitals.operations import gate as _gate
 
     return _gate(
         current,

From ad7db22ba3f0a423fb5cca915d15d516823e07ab Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:00:55 +0530
Subject: [PATCH 137/439] test: keep lightweight operations off full pipeline

---
 tests/test_operation_import_boundaries.py | 38 +++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 tests/test_operation_import_boundaries.py

diff --git a/tests/test_operation_import_boundaries.py b/tests/test_operation_import_boundaries.py
new file mode 100644
index 0000000..111e982
--- /dev/null
+++ b/tests/test_operation_import_boundaries.py
@@ -0,0 +1,38 @@
+import subprocess
+import sys
+
+
+def test_contract_and_drift_operations_do_not_load_full_pipeline():
+    code = r'''
+import sys
+import pandas as pd
+import framevitals as fv
+
+reference = pd.DataFrame({"x": range(30), "label": ["a", "b"] * 15})
+current = reference.copy()
+
+contract = fv.infer_contract(reference)
+validation = fv.validate(current, contract)
+drift = fv.compare(reference, current)
+gate = fv.gate(current, reference=reference, contract=contract)
+
+assert validation["status"] in {"pass", "warn"}
+assert drift["available"] is True
+assert gate["passed"] is True
+
+for name in (
+    "framevitals.pipeline",
+    "framevitals.anomaly_ensemble",
+    "framevitals.deep_statistics_v2",
+    "framevitals.model_leaderboard",
+):
+    assert name not in sys.modules, name
+'''
+    completed = subprocess.run(
+        [sys.executable, "-c", code],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+    assert completed.returncode == 0, completed.stderr

From 37e295adaf2a8417e0f53339ada6f2fa90169481 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:10:44 +0530
Subject: [PATCH 138/439] feat(native): add mergeable bounded sketches

---
 rust/framevitals-core/src/sketches.rs | 512 ++++++++++++++++++++++++++
 1 file changed, 512 insertions(+)
 create mode 100644 rust/framevitals-core/src/sketches.rs

diff --git a/rust/framevitals-core/src/sketches.rs b/rust/framevitals-core/src/sketches.rs
new file mode 100644
index 0000000..8595c0a
--- /dev/null
+++ b/rust/framevitals-core/src/sketches.rs
@@ -0,0 +1,512 @@
+//! Mergeable bounded-memory sketches used by the native FrameVitals profiler.
+//!
+//! These structures intentionally avoid retaining raw observations. Each sketch
+//! can be built per Arrow batch/partition and merged later, which makes the same
+//! semantics usable for local streaming, parallel scans, and distributed jobs.
+
+use std::cmp::Ordering;
+use std::collections::BTreeMap;
+
+const DEFAULT_ZERO_THRESHOLD: f64 = 1.0e-12;
+
+#[inline]
+pub(crate) fn mix64(mut value: u64) -> u64 {
+    value = value.wrapping_add(0x9E37_79B9_7F4A_7C15);
+    value = (value ^ (value >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
+    value = (value ^ (value >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
+    value ^ (value >> 31)
+}
+
+#[inline]
+fn canonical_f64_bits(value: f64) -> u64 {
+    if value == 0.0 {
+        0
+    } else {
+        value.to_bits()
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct HyperLogLog {
+    precision: u8,
+    registers: Vec<u8>,
+}
+
+impl HyperLogLog {
+    pub fn new(precision: u8) -> Self {
+        assert!((4..=16).contains(&precision), "HLL precision must be 4..=16");
+        Self {
+            precision,
+            registers: vec![0; 1usize << precision],
+        }
+    }
+
+    pub fn observe_f64(&mut self, value: f64) {
+        if !value.is_finite() {
+            return;
+        }
+        self.observe_hash(mix64(canonical_f64_bits(value)));
+    }
+
+    pub fn observe_hash(&mut self, hash: u64) {
+        let index = (hash >> (64 - self.precision)) as usize;
+        let remainder = hash << self.precision;
+        let max_rank = 64 - u32::from(self.precision) + 1;
+        let rank = (remainder.leading_zeros() + 1).min(max_rank) as u8;
+        self.registers[index] = self.registers[index].max(rank);
+    }
+
+    #[must_use]
+    pub fn merge(mut self, other: Self) -> Self {
+        assert_eq!(self.precision, other.precision, "HLL precision mismatch");
+        for (left, right) in self.registers.iter_mut().zip(other.registers) {
+            *left = (*left).max(right);
+        }
+        self
+    }
+
+    #[must_use]
+    pub fn estimate(&self) -> f64 {
+        let m = self.registers.len() as f64;
+        let alpha = match self.registers.len() {
+            16 => 0.673,
+            32 => 0.697,
+            64 => 0.709,
+            _ => 0.7213 / (1.0 + 1.079 / m),
+        };
+        let harmonic_sum: f64 = self
+            .registers
+            .iter()
+            .map(|register| 2.0_f64.powi(-i32::from(*register)))
+            .sum();
+        let raw = alpha * m * m / harmonic_sum;
+        let zero_registers = self.registers.iter().filter(|&&value| value == 0).count();
+
+        if raw <= 2.5 * m && zero_registers > 0 {
+            m * (m / zero_registers as f64).ln()
+        } else {
+            raw
+        }
+    }
+
+    pub fn precision(&self) -> u8 {
+        self.precision
+    }
+
+    pub fn bytes_used(&self) -> usize {
+        self.registers.len()
+    }
+}
+
+impl Default for HyperLogLog {
+    fn default() -> Self {
+        Self::new(12)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct LogQuantileSketch {
+    relative_accuracy: f64,
+    log_gamma: f64,
+    zero_threshold: f64,
+    negative: BTreeMap<i32, u64>,
+    positive: BTreeMap<i32, u64>,
+    zero_count: u64,
+    count: u64,
+}
+
+impl LogQuantileSketch {
+    pub fn new(relative_accuracy: f64) -> Self {
+        assert!(
+            relative_accuracy > 0.0 && relative_accuracy < 1.0,
+            "relative accuracy must be between 0 and 1"
+        );
+        let gamma = (1.0 + relative_accuracy) / (1.0 - relative_accuracy);
+        Self {
+            relative_accuracy,
+            log_gamma: gamma.ln(),
+            zero_threshold: DEFAULT_ZERO_THRESHOLD,
+            negative: BTreeMap::new(),
+            positive: BTreeMap::new(),
+            zero_count: 0,
+            count: 0,
+        }
+    }
+
+    fn key(&self, magnitude: f64) -> i32 {
+        (magnitude.ln() / self.log_gamma).floor() as i32
+    }
+
+    fn representative(&self, key: i32) -> f64 {
+        ((key as f64 + 0.5) * self.log_gamma).exp()
+    }
+
+    pub fn observe(&mut self, value: f64) {
+        if !value.is_finite() {
+            return;
+        }
+        self.count += 1;
+        if value.abs() <= self.zero_threshold {
+            self.zero_count += 1;
+        } else if value < 0.0 {
+            *self.negative.entry(self.key(-value)).or_insert(0) += 1;
+        } else {
+            *self.positive.entry(self.key(value)).or_insert(0) += 1;
+        }
+    }
+
+    #[must_use]
+    pub fn merge(mut self, other: Self) -> Self {
+        assert!(
+            (self.relative_accuracy - other.relative_accuracy).abs() <= f64::EPSILON,
+            "quantile sketch accuracy mismatch"
+        );
+        self.count += other.count;
+        self.zero_count += other.zero_count;
+        for (key, count) in other.negative {
+            *self.negative.entry(key).or_insert(0) += count;
+        }
+        for (key, count) in other.positive {
+            *self.positive.entry(key).or_insert(0) += count;
+        }
+        self
+    }
+
+    pub fn quantile(&self, q: f64) -> Option<f64> {
+        if self.count == 0 || !(0.0..=1.0).contains(&q) {
+            return None;
+        }
+        let target = (q * (self.count - 1) as f64).floor() as u64;
+        let mut seen = 0_u64;
+
+        for (key, count) in self.negative.iter().rev() {
+            if target < seen + count {
+                return Some(-self.representative(*key));
+            }
+            seen += count;
+        }
+        if target < seen + self.zero_count {
+            return Some(0.0);
+        }
+        seen += self.zero_count;
+        for (key, count) in &self.positive {
+            if target < seen + count {
+                return Some(self.representative(*key));
+            }
+            seen += count;
+        }
+        self.positive
+            .last_key_value()
+            .map(|(key, _)| self.representative(*key))
+            .or_else(|| {
+                self.negative
+                    .first_key_value()
+                    .map(|(key, _)| -self.representative(*key))
+            })
+            .or(Some(0.0))
+    }
+
+    pub fn count(&self) -> u64 {
+        self.count
+    }
+
+    pub fn bin_count(&self) -> usize {
+        self.negative.len() + self.positive.len() + usize::from(self.zero_count > 0)
+    }
+
+    pub fn relative_accuracy(&self) -> f64 {
+        self.relative_accuracy
+    }
+}
+
+impl Default for LogQuantileSketch {
+    fn default() -> Self {
+        Self::new(0.01)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct HeavyHittersSketch {
+    capacity: usize,
+    counters: BTreeMap<u64, u64>,
+}
+
+impl HeavyHittersSketch {
+    pub fn new(capacity: usize) -> Self {
+        assert!(capacity > 0, "heavy-hitter capacity must be positive");
+        Self {
+            capacity,
+            counters: BTreeMap::new(),
+        }
+    }
+
+    pub fn observe_f64(&mut self, value: f64) {
+        if value.is_finite() {
+            self.observe_weighted(canonical_f64_bits(value), 1);
+        }
+    }
+
+    pub fn observe_weighted(&mut self, key: u64, weight: u64) {
+        if weight == 0 {
+            return;
+        }
+        if let Some(counter) = self.counters.get_mut(&key) {
+            *counter += weight;
+            return;
+        }
+        if self.counters.len() < self.capacity {
+            self.counters.insert(key, weight);
+            return;
+        }
+
+        let minimum = self.counters.values().copied().min().unwrap_or(0);
+        let decrement = minimum.min(weight);
+        self.counters.retain(|_, count| {
+            *count -= decrement;
+            *count > 0
+        });
+        let remaining = weight - decrement;
+        if remaining > 0 {
+            self.counters.insert(key, remaining);
+        }
+    }
+
+    #[must_use]
+    pub fn merge(mut self, other: Self) -> Self {
+        assert_eq!(self.capacity, other.capacity, "heavy-hitter capacity mismatch");
+        for (key, count) in other.counters {
+            self.observe_weighted(key, count);
+        }
+        self
+    }
+
+    pub fn candidates(&self) -> Vec<(f64, u64)> {
+        let mut values: Vec<(f64, u64)> = self
+            .counters
+            .iter()
+            .map(|(bits, count)| (f64::from_bits(*bits), *count))
+            .collect();
+        values.sort_by(|left, right| {
+            right
+                .1
+                .cmp(&left.1)
+                .then_with(|| left.0.total_cmp(&right.0))
+        });
+        values
+    }
+
+    pub fn capacity(&self) -> usize {
+        self.capacity
+    }
+}
+
+impl Default for HeavyHittersSketch {
+    fn default() -> Self {
+        Self::new(32)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct ReservoirEntry {
+    pub priority: u64,
+    pub value: f64,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct PriorityReservoir {
+    capacity: usize,
+    entries: Vec<ReservoirEntry>,
+}
+
+impl PriorityReservoir {
+    pub fn new(capacity: usize) -> Self {
+        assert!(capacity > 0, "reservoir capacity must be positive");
+        Self {
+            capacity,
+            entries: Vec::with_capacity(capacity),
+        }
+    }
+
+    pub fn observe(&mut self, value: f64, stream_id: u64, sequence: u64) {
+        if !value.is_finite() {
+            return;
+        }
+        let value_hash = mix64(canonical_f64_bits(value));
+        let priority = mix64(value_hash ^ mix64(stream_id) ^ mix64(sequence));
+        let entry = ReservoirEntry { priority, value };
+
+        if self.entries.len() < self.capacity {
+            self.entries.push(entry);
+            return;
+        }
+        if let Some((index, largest)) = self
+            .entries
+            .iter()
+            .enumerate()
+            .max_by_key(|(_, current)| current.priority)
+        {
+            if priority < largest.priority {
+                self.entries[index] = entry;
+            }
+        }
+    }
+
+    #[must_use]
+    pub fn merge(mut self, other: Self) -> Self {
+        assert_eq!(self.capacity, other.capacity, "reservoir capacity mismatch");
+        self.entries.extend(other.entries);
+        self.entries.sort_by_key(|entry| entry.priority);
+        self.entries.truncate(self.capacity);
+        self
+    }
+
+    pub fn values(&self) -> Vec<f64> {
+        let mut entries = self.entries.clone();
+        entries.sort_by(|left, right| {
+            left.value
+                .partial_cmp(&right.value)
+                .unwrap_or(Ordering::Equal)
+                .then_with(|| left.priority.cmp(&right.priority))
+        });
+        entries.into_iter().map(|entry| entry.value).collect()
+    }
+
+    pub fn len(&self) -> usize {
+        self.entries.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.entries.is_empty()
+    }
+
+    pub fn capacity(&self) -> usize {
+        self.capacity
+    }
+}
+
+impl Default for PriorityReservoir {
+    fn default() -> Self {
+        Self::new(256)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct NumericSketchState {
+    pub cardinality: HyperLogLog,
+    pub quantiles: LogQuantileSketch,
+    pub heavy_hitters: HeavyHittersSketch,
+    pub reservoir: PriorityReservoir,
+}
+
+impl NumericSketchState {
+    pub fn observe(&mut self, value: f64, stream_id: u64, sequence: u64) {
+        if !value.is_finite() {
+            return;
+        }
+        self.cardinality.observe_f64(value);
+        self.quantiles.observe(value);
+        self.heavy_hitters.observe_f64(value);
+        self.reservoir.observe(value, stream_id, sequence);
+    }
+
+    #[must_use]
+    pub fn merge(self, other: Self) -> Self {
+        Self {
+            cardinality: self.cardinality.merge(other.cardinality),
+            quantiles: self.quantiles.merge(other.quantiles),
+            heavy_hitters: self.heavy_hitters.merge(other.heavy_hitters),
+            reservoir: self.reservoir.merge(other.reservoir),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        HeavyHittersSketch, HyperLogLog, LogQuantileSketch, NumericSketchState,
+        PriorityReservoir,
+    };
+
+    #[test]
+    fn hll_estimates_cardinality_and_merges_partitions() {
+        let mut left = HyperLogLog::new(12);
+        let mut right = HyperLogLog::new(12);
+        for value in 0..10_000 {
+            if value < 5_000 {
+                left.observe_f64(value as f64);
+            } else {
+                right.observe_f64(value as f64);
+            }
+        }
+        let estimate = left.merge(right).estimate();
+        assert!((estimate - 10_000.0).abs() / 10_000.0 < 0.08);
+    }
+
+    #[test]
+    fn logarithmic_quantiles_are_mergeable_and_bounded() {
+        let mut left = LogQuantileSketch::new(0.01);
+        let mut right = LogQuantileSketch::new(0.01);
+        for value in 1..=10_000 {
+            if value <= 5_000 {
+                left.observe(value as f64);
+            } else {
+                right.observe(value as f64);
+            }
+        }
+        let merged = left.merge(right);
+        let median = merged.quantile(0.5).unwrap();
+        assert!((median - 5_000.0).abs() / 5_000.0 < 0.03);
+        assert!(merged.bin_count() < 1_000);
+    }
+
+    #[test]
+    fn heavy_hitters_retain_dominant_candidates_after_merge() {
+        let mut left = HeavyHittersSketch::new(8);
+        let mut right = HeavyHittersSketch::new(8);
+        for _ in 0..100 {
+            left.observe_f64(7.0);
+        }
+        for value in 0..50 {
+            left.observe_f64(value as f64);
+            right.observe_f64((value + 50) as f64);
+        }
+        for _ in 0..80 {
+            right.observe_f64(7.0);
+        }
+        let candidates = left.merge(right).candidates();
+        assert_eq!(candidates.first().map(|item| item.0), Some(7.0));
+    }
+
+    #[test]
+    fn priority_reservoir_is_bounded_and_partition_mergeable() {
+        let mut left = PriorityReservoir::new(32);
+        let mut right = PriorityReservoir::new(32);
+        for value in 0..1_000_u64 {
+            if value < 500 {
+                left.observe(value as f64, 1, value);
+            } else {
+                right.observe(value as f64, 2, value - 500);
+            }
+        }
+        let merged = left.merge(right);
+        assert_eq!(merged.len(), 32);
+        assert_eq!(merged.values().len(), 32);
+    }
+
+    #[test]
+    fn combined_numeric_sketch_merges_without_raw_values() {
+        let mut left = NumericSketchState::default();
+        let mut right = NumericSketchState::default();
+        for value in 0..2_000_u64 {
+            if value < 1_000 {
+                left.observe(value as f64, 11, value);
+            } else {
+                right.observe(value as f64, 12, value - 1_000);
+            }
+        }
+        let merged = left.merge(right);
+        assert!(merged.cardinality.estimate() > 1_800.0);
+        assert!(merged.quantiles.quantile(0.5).unwrap() > 900.0);
+        assert_eq!(merged.reservoir.len(), 256);
+    }
+}

From ee85d1d97b68afa57120a06ec5b88dc90c72ce9f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:11:22 +0530
Subject: [PATCH 139/439] feat(native): expose sketch primitives

---
 rust/framevitals-core/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/rust/framevitals-core/src/lib.rs b/rust/framevitals-core/src/lib.rs
index 9f85051..5b7e1c3 100644
--- a/rust/framevitals-core/src/lib.rs
+++ b/rust/framevitals-core/src/lib.rs
@@ -7,6 +7,7 @@
 
 #[cfg(feature = "arrow")]
 pub mod arrow_scan;
+pub mod sketches;
 
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub struct NumericState {

From 20a7e7624125a4b757d3e596b017251959b7cb2c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:12:11 +0530
Subject: [PATCH 140/439] feat(native): add fused Arrow numeric profiler

---
 rust/framevitals-core/src/fused_profile.rs | 201 +++++++++++++++++++++
 1 file changed, 201 insertions(+)
 create mode 100644 rust/framevitals-core/src/fused_profile.rs

diff --git a/rust/framevitals-core/src/fused_profile.rs b/rust/framevitals-core/src/fused_profile.rs
new file mode 100644
index 0000000..798b060
--- /dev/null
+++ b/rust/framevitals-core/src/fused_profile.rs
@@ -0,0 +1,201 @@
+//! Fused Arrow numeric profiling.
+//!
+//! One pass over each supported numeric Arrow column updates exact moments and
+//! bounded sketches together. This avoids a separate full-column scan for every
+//! statistic and produces states that can be merged across batches/partitions.
+
+use std::collections::{BTreeMap, BTreeSet};
+
+use arrow_array::{
+    Array, Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, UInt32Array,
+    UInt64Array,
+};
+use arrow_schema::DataType;
+
+use crate::sketches::NumericSketchState;
+use crate::NumericState;
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct NumericProfileState {
+    pub moments: NumericState,
+    pub sketches: NumericSketchState,
+}
+
+impl NumericProfileState {
+    pub fn observe(&mut self, value: Option<f64>, stream_id: u64, sequence: u64) {
+        self.moments.observe(value);
+        if let Some(value) = value {
+            self.sketches.observe(value, stream_id, sequence);
+        }
+    }
+
+    #[must_use]
+    pub fn merge(self, other: Self) -> Self {
+        Self {
+            moments: self.moments.merge(other.moments),
+            sketches: self.sketches.merge(other.sketches),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct BatchProfileState {
+    pub rows: u64,
+    pub profiles: BTreeMap<String, NumericProfileState>,
+    pub skipped_columns: BTreeSet<String>,
+}
+
+impl BatchProfileState {
+    #[must_use]
+    pub fn merge(mut self, other: Self) -> Self {
+        self.rows += other.rows;
+        self.skipped_columns.extend(other.skipped_columns);
+        for (name, profile) in other.profiles {
+            self.profiles
+                .entry(name)
+                .and_modify(|current| *current = current.clone().merge(profile.clone()))
+                .or_insert(profile);
+        }
+        self
+    }
+}
+
+macro_rules! scan_profile_primitive {
+    ($array:expr, $stream_id:expr) => {{
+        let array = $array;
+        let mut profile = NumericProfileState::default();
+        for index in 0..array.len() {
+            if array.is_null(index) {
+                profile.observe(None, $stream_id, index as u64);
+            } else {
+                profile.observe(Some(array.value(index) as f64), $stream_id, index as u64);
+            }
+        }
+        profile
+    }};
+}
+
+/// Scan supported primitive numeric columns in one fused pass.
+///
+/// `partition_id` must be stable and distinct for independently scanned
+/// partitions when deterministic reservoirs are expected to merge without
+/// priority collisions. Moments and the other sketches do not depend on it.
+pub fn profile_record_batch(batch: &RecordBatch, partition_id: u64) -> BatchProfileState {
+    let mut output = BatchProfileState {
+        rows: batch.num_rows() as u64,
+        ..BatchProfileState::default()
+    };
+
+    for (index, field) in batch.schema().fields().iter().enumerate() {
+        let name = field.name().clone();
+        let array = batch.column(index);
+        let stream_id = partition_id
+            .wrapping_mul(0x9E37_79B9_7F4A_7C15)
+            .wrapping_add(index as u64);
+
+        let profile = match field.data_type() {
+            DataType::Float64 => array
+                .as_any()
+                .downcast_ref::<Float64Array>()
+                .map(|values| scan_profile_primitive!(values, stream_id)),
+            DataType::Float32 => array
+                .as_any()
+                .downcast_ref::<Float32Array>()
+                .map(|values| scan_profile_primitive!(values, stream_id)),
+            DataType::Int64 => array
+                .as_any()
+                .downcast_ref::<Int64Array>()
+                .map(|values| scan_profile_primitive!(values, stream_id)),
+            DataType::Int32 => array
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .map(|values| scan_profile_primitive!(values, stream_id)),
+            DataType::UInt64 => array
+                .as_any()
+                .downcast_ref::<UInt64Array>()
+                .map(|values| scan_profile_primitive!(values, stream_id)),
+            DataType::UInt32 => array
+                .as_any()
+                .downcast_ref::<UInt32Array>()
+                .map(|values| scan_profile_primitive!(values, stream_id)),
+            _ => None,
+        };
+
+        match profile {
+            Some(profile) => {
+                output.profiles.insert(name, profile);
+            }
+            None => {
+                output.skipped_columns.insert(name);
+            }
+        }
+    }
+
+    output
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::{ArrayRef, Float64Array, Int64Array, RecordBatch, StringArray};
+
+    use super::profile_record_batch;
+
+    fn batch(values: Vec<Option<f64>>, counts: Vec<Option<i64>>) -> RecordBatch {
+        let rows = values.len();
+        assert_eq!(counts.len(), rows);
+        let labels: Vec<Option<&str>> = (0..rows)
+            .map(|index| if index % 2 == 0 { Some("x") } else { None })
+            .collect();
+        RecordBatch::try_from_iter(vec![
+            ("value", Arc::new(Float64Array::from(values)) as ArrayRef),
+            ("count", Arc::new(Int64Array::from(counts)) as ArrayRef),
+            ("label", Arc::new(StringArray::from(labels)) as ArrayRef),
+        ])
+        .expect("valid batch")
+    }
+
+    #[test]
+    fn fused_profile_updates_moments_and_sketches_together() {
+        let batch = batch(
+            vec![Some(1.0), Some(2.0), None, Some(2.0), Some(f64::INFINITY)],
+            vec![Some(1), Some(2), Some(3), Some(4), Some(5)],
+        );
+        let state = profile_record_batch(&batch, 7);
+        let value = state.profiles.get("value").unwrap();
+
+        assert_eq!(state.rows, 5);
+        assert_eq!(value.moments.count, 3);
+        assert_eq!(value.moments.missing, 1);
+        assert_eq!(value.moments.infinite, 1);
+        assert!((value.moments.mean - 5.0 / 3.0).abs() < 1e-12);
+        assert!(value.sketches.cardinality.estimate() > 1.5);
+        assert_eq!(value.sketches.quantiles.count(), 3);
+        assert!(!value.sketches.reservoir.is_empty());
+        assert!(state.skipped_columns.contains("label"));
+    }
+
+    #[test]
+    fn fused_partition_merge_preserves_compact_profile_semantics() {
+        let left = batch(
+            (0..1_000).map(|value| Some(value as f64)).collect(),
+            (0..1_000).map(|value| Some(value as i64)).collect(),
+        );
+        let right = batch(
+            (1_000..2_000).map(|value| Some(value as f64)).collect(),
+            (1_000..2_000).map(|value| Some(value as i64)).collect(),
+        );
+
+        let merged = profile_record_batch(&left, 1).merge(profile_record_batch(&right, 2));
+        let value = merged.profiles.get("value").unwrap();
+
+        assert_eq!(merged.rows, 2_000);
+        assert_eq!(value.moments.count, 2_000);
+        assert!((value.moments.mean - 999.5).abs() < 1e-12);
+        assert!(value.sketches.cardinality.estimate() > 1_800.0);
+        let median = value.sketches.quantiles.quantile(0.5).unwrap();
+        assert!(median > 900.0 && median < 1_100.0);
+        assert_eq!(value.sketches.reservoir.len(), 256);
+    }
+}

From 74e599b74a06eabe984d1024ad20ff4f5661f786 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:12:49 +0530
Subject: [PATCH 141/439] feat(native): expose fused profiler

---
 rust/framevitals-core/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/rust/framevitals-core/src/lib.rs b/rust/framevitals-core/src/lib.rs
index 5b7e1c3..eff6361 100644
--- a/rust/framevitals-core/src/lib.rs
+++ b/rust/framevitals-core/src/lib.rs
@@ -7,6 +7,8 @@
 
 #[cfg(feature = "arrow")]
 pub mod arrow_scan;
+#[cfg(feature = "arrow")]
+pub mod fused_profile;
 pub mod sketches;
 
 #[derive(Debug, Clone, Copy, PartialEq)]

From be8fad6853ef0f87cdaae064c88197f8b5a59aa3 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:13:30 +0530
Subject: [PATCH 142/439] feat(native): add PyO3 bridge crate

---
 rust/framevitals-py/Cargo.toml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 rust/framevitals-py/Cargo.toml

diff --git a/rust/framevitals-py/Cargo.toml b/rust/framevitals-py/Cargo.toml
new file mode 100644
index 0000000..023eb27
--- /dev/null
+++ b/rust/framevitals-py/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "framevitals-py"
+version = "0.1.0"
+edition = "2021"
+description = "PyO3 bridge for FrameVitals native kernels"
+license = "MIT"
+publish = false
+
+[lib]
+name = "_native"
+path = "src/lib.rs"
+crate-type = ["cdylib", "rlib"]
+
+[features]
+default = []
+extension-module = ["pyo3/extension-module"]
+
+[dependencies]
+framevitals-core = { path = "../framevitals-core" }
+pyo3 = { version = "0.29", features = ["abi3-py311"] }

From 47dfc86d2b3beb30c8189cdbfa4d228ef50e902b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:13:50 +0530
Subject: [PATCH 143/439] feat(native): expose numeric profiling through PyO3

---
 rust/framevitals-py/src/lib.rs | 122 +++++++++++++++++++++++++++++++++
 1 file changed, 122 insertions(+)
 create mode 100644 rust/framevitals-py/src/lib.rs

diff --git a/rust/framevitals-py/src/lib.rs b/rust/framevitals-py/src/lib.rs
new file mode 100644
index 0000000..082eeb5
--- /dev/null
+++ b/rust/framevitals-py/src/lib.rs
@@ -0,0 +1,122 @@
+//! Python bindings for FrameVitals native kernels.
+
+use framevitals_core::sketches::NumericSketchState;
+use framevitals_core::NumericState;
+use pyo3::buffer::PyBuffer;
+use pyo3::exceptions::PyBufferError;
+use pyo3::prelude::*;
+use pyo3::types::PyDict;
+
+fn contiguous_f64_slice<'py>(
+    py: Python<'py>,
+    values: &Bound<'py, PyAny>,
+) -> PyResult<(PyBuffer<f64>, &'py [pyo3::buffer::ReadOnlyCell<f64>])> {
+    let buffer = PyBuffer::<f64>::get(values)?;
+    if buffer.dimensions() != 1 {
+        return Err(PyBufferError::new_err(
+            "FrameVitals native numeric kernels require a 1D float64 buffer.",
+        ));
+    }
+    let slice = buffer.as_slice(py).ok_or_else(|| {
+        PyBufferError::new_err(
+            "FrameVitals native numeric kernels require a C-contiguous float64 buffer.",
+        )
+    })?;
+
+    // The slice lifetime is tied to both Python and the owned PyBuffer. Returning
+    // both keeps the exporter alive for the duration of the caller's scan.
+    Ok((buffer, slice))
+}
+
+fn exact_state_dict<'py>(
+    py: Python<'py>,
+    state: &NumericState,
+) -> PyResult<Bound<'py, PyDict>> {
+    let payload = PyDict::new(py);
+    payload.set_item("count", state.count)?;
+    payload.set_item("missing", state.missing)?;
+    payload.set_item("infinite", state.infinite)?;
+    payload.set_item(
+        "mean",
+        if state.count > 0 { Some(state.mean) } else { None },
+    )?;
+    payload.set_item("variance", state.variance())?;
+    payload.set_item("std", state.standard_deviation())?;
+    payload.set_item("minimum", state.minimum)?;
+    payload.set_item("maximum", state.maximum)?;
+    Ok(payload)
+}
+
+#[pyfunction]
+fn numeric_state_f64(py: Python<'_>, values: &Bound<'_, PyAny>) -> PyResult<Py<PyDict>> {
+    let (buffer, slice) = contiguous_f64_slice(py, values)?;
+    let mut state = NumericState::default();
+    for value in slice {
+        state.observe(Some(value.get()));
+    }
+    let payload = exact_state_dict(py, &state)?;
+    payload.set_item("backend", "rust")?;
+    payload.set_item("observations", buffer.item_count())?;
+    Ok(payload.unbind())
+}
+
+#[pyfunction]
+#[pyo3(signature = (values, stream_id = 0))]
+fn numeric_profile_f64(
+    py: Python<'_>,
+    values: &Bound<'_, PyAny>,
+    stream_id: u64,
+) -> PyResult<Py<PyDict>> {
+    let (buffer, slice) = contiguous_f64_slice(py, values)?;
+    let mut state = NumericState::default();
+    let mut sketches = NumericSketchState::default();
+
+    for (index, value) in slice.iter().enumerate() {
+        let value = value.get();
+        state.observe(Some(value));
+        sketches.observe(value, stream_id, index as u64);
+    }
+
+    let payload = exact_state_dict(py, &state)?;
+    payload.set_item("backend", "rust")?;
+    payload.set_item("observations", buffer.item_count())?;
+    payload.set_item(
+        "cardinality_estimate",
+        sketches.cardinality.estimate().round() as u64,
+    )?;
+
+    let quantiles = PyDict::new(py);
+    for (name, q) in [
+        ("p01", 0.01),
+        ("p05", 0.05),
+        ("p25", 0.25),
+        ("p50", 0.50),
+        ("p75", 0.75),
+        ("p95", 0.95),
+        ("p99", 0.99),
+    ] {
+        quantiles.set_item(name, sketches.quantiles.quantile(q))?;
+    }
+    quantiles.set_item(
+        "relative_accuracy",
+        sketches.quantiles.relative_accuracy(),
+    )?;
+    payload.set_item("quantiles", quantiles)?;
+    payload.set_item("heavy_hitters", sketches.heavy_hitters.candidates())?;
+    payload.set_item("reservoir", sketches.reservoir.values())?;
+    Ok(payload.unbind())
+}
+
+#[pyfunction]
+fn backend_info() -> (&'static str, &'static str) {
+    ("rust", env!("CARGO_PKG_VERSION"))
+}
+
+#[pymodule]
+fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> {
+    module.add_function(wrap_pyfunction!(numeric_state_f64, module)?)?;
+    module.add_function(wrap_pyfunction!(numeric_profile_f64, module)?)?;
+    module.add_function(wrap_pyfunction!(backend_info, module)?)?;
+    module.add("__version__", env!("CARGO_PKG_VERSION"))?;
+    Ok(())
+}

From bcbb1babadfa36fd07534c386fa42b87a62c49f8 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:14:17 +0530
Subject: [PATCH 144/439] fix(native): keep Python buffer alive during scans

---
 rust/framevitals-py/src/lib.rs | 52 +++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 23 deletions(-)

diff --git a/rust/framevitals-py/src/lib.rs b/rust/framevitals-py/src/lib.rs
index 082eeb5..a28e194 100644
--- a/rust/framevitals-py/src/lib.rs
+++ b/rust/framevitals-py/src/lib.rs
@@ -7,27 +7,6 @@ use pyo3::exceptions::PyBufferError;
 use pyo3::prelude::*;
 use pyo3::types::PyDict;
 
-fn contiguous_f64_slice<'py>(
-    py: Python<'py>,
-    values: &Bound<'py, PyAny>,
-) -> PyResult<(PyBuffer<f64>, &'py [pyo3::buffer::ReadOnlyCell<f64>])> {
-    let buffer = PyBuffer::<f64>::get(values)?;
-    if buffer.dimensions() != 1 {
-        return Err(PyBufferError::new_err(
-            "FrameVitals native numeric kernels require a 1D float64 buffer.",
-        ));
-    }
-    let slice = buffer.as_slice(py).ok_or_else(|| {
-        PyBufferError::new_err(
-            "FrameVitals native numeric kernels require a C-contiguous float64 buffer.",
-        )
-    })?;
-
-    // The slice lifetime is tied to both Python and the owned PyBuffer. Returning
-    // both keeps the exporter alive for the duration of the caller's scan.
-    Ok((buffer, slice))
-}
-
 fn exact_state_dict<'py>(
     py: Python<'py>,
     state: &NumericState,
@@ -47,9 +26,30 @@ fn exact_state_dict<'py>(
     Ok(payload)
 }
 
+fn checked_buffer<'py>(values: &Bound<'py, PyAny>) -> PyResult<PyBuffer<f64>> {
+    let buffer = PyBuffer::<f64>::get(values)?;
+    if buffer.dimensions() != 1 {
+        return Err(PyBufferError::new_err(
+            "FrameVitals native numeric kernels require a 1D float64 buffer.",
+        ));
+    }
+    if !buffer.is_c_contiguous() {
+        return Err(PyBufferError::new_err(
+            "FrameVitals native numeric kernels require a C-contiguous float64 buffer.",
+        ));
+    }
+    Ok(buffer)
+}
+
 #[pyfunction]
 fn numeric_state_f64(py: Python<'_>, values: &Bound<'_, PyAny>) -> PyResult<Py<PyDict>> {
-    let (buffer, slice) = contiguous_f64_slice(py, values)?;
+    let buffer = checked_buffer(values)?;
+    let slice = buffer.as_slice(py).ok_or_else(|| {
+        PyBufferError::new_err(
+            "FrameVitals could not borrow the supplied float64 buffer as a contiguous slice.",
+        )
+    })?;
+
     let mut state = NumericState::default();
     for value in slice {
         state.observe(Some(value.get()));
@@ -67,7 +67,13 @@ fn numeric_profile_f64(
     values: &Bound<'_, PyAny>,
     stream_id: u64,
 ) -> PyResult<Py<PyDict>> {
-    let (buffer, slice) = contiguous_f64_slice(py, values)?;
+    let buffer = checked_buffer(values)?;
+    let slice = buffer.as_slice(py).ok_or_else(|| {
+        PyBufferError::new_err(
+            "FrameVitals could not borrow the supplied float64 buffer as a contiguous slice.",
+        )
+    })?;
+
     let mut state = NumericState::default();
     let mut sketches = NumericSketchState::default();
 

From 23a94128aaf3b793b95cb27a0cff3c5830e931d2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:14:29 +0530
Subject: [PATCH 145/439] build(native): include Python bridge in Rust
 workspace

---
 rust/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 6fbe462..77fdd81 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -1,3 +1,3 @@
 [workspace]
-members = ["framevitals-core"]
+members = ["framevitals-core", "framevitals-py"]
 resolver = "2"

From 80649aa4f5b799b4d41716436e8480eb30326337 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:15:20 +0530
Subject: [PATCH 146/439] feat: add numeric backend router

---
 src/framevitals/backends.py | 147 ++++++++++++++++++++++++++++++++++++
 1 file changed, 147 insertions(+)
 create mode 100644 src/framevitals/backends.py

diff --git a/src/framevitals/backends.py b/src/framevitals/backends.py
new file mode 100644
index 0000000..3af0ecb
--- /dev/null
+++ b/src/framevitals/backends.py
@@ -0,0 +1,147 @@
+"""Backend routing for FrameVitals numeric kernels.
+
+The router keeps optional native/GPU dependencies behind lazy imports. Normal
+installations remain NumPy-first, while a compiled ``framevitals._native``
+extension can transparently accelerate compatible numeric scans.
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+from importlib.util import find_spec
+import os
+from typing import Any, Literal
+
+import numpy as np
+import pandas as pd
+
+
+BackendName = Literal["auto", "numpy", "rust"]
+_VALID_BACKENDS = {"auto", "numpy", "rust"}
+
+
+def native_available() -> bool:
+    """Return whether the optional FrameVitals native extension is importable."""
+    return find_spec("framevitals._native") is not None
+
+
+def resolve_numeric_backend(requested: str | None = None) -> Literal["numpy", "rust"]:
+    """Resolve the backend for exact/streaming numeric primitives.
+
+    ``FRAMEVITALS_BACKEND`` may set ``auto``, ``numpy`` or ``rust``. Explicit
+    function arguments take precedence. Requesting Rust without the extension
+    installed is an error; ``auto`` always falls back safely to NumPy.
+    """
+    value = (requested or os.getenv("FRAMEVITALS_BACKEND", "auto")).strip().lower()
+    if value not in _VALID_BACKENDS:
+        raise ValueError(
+            "Unknown FrameVitals numeric backend "
+            f"'{value}'. Choose from: auto, numpy, rust."
+        )
+    if value == "numpy":
+        return "numpy"
+    if value == "rust":
+        if not native_available():
+            raise RuntimeError(
+                "The Rust backend was requested but framevitals._native is not installed."
+            )
+        return "rust"
+    return "rust" if native_available() else "numpy"
+
+
+def _float64_array(values: pd.Series | np.ndarray | list[Any]) -> np.ndarray:
+    if isinstance(values, pd.Series):
+        array = pd.to_numeric(values, errors="coerce").to_numpy(
+            dtype="float64",
+            na_value=np.nan,
+        )
+    else:
+        array = np.asarray(values, dtype=np.float64)
+    if array.ndim != 1:
+        raise ValueError("FrameVitals numeric kernels require one-dimensional input.")
+    return np.ascontiguousarray(array, dtype=np.float64)
+
+
+def _numpy_numeric_state(array: np.ndarray) -> dict[str, Any]:
+    missing = int(np.isnan(array).sum())
+    infinite = int(np.isinf(array).sum())
+    finite = array[np.isfinite(array)]
+    if finite.size == 0:
+        return {
+            "backend": "numpy",
+            "observations": int(array.size),
+            "count": 0,
+            "missing": missing,
+            "infinite": infinite,
+            "mean": None,
+            "variance": None,
+            "std": None,
+            "minimum": None,
+            "maximum": None,
+        }
+
+    mean = float(finite.mean())
+    variance = float(finite.var(ddof=1)) if finite.size >= 2 else None
+    return {
+        "backend": "numpy",
+        "observations": int(array.size),
+        "count": int(finite.size),
+        "missing": missing,
+        "infinite": infinite,
+        "mean": mean,
+        "variance": variance,
+        "std": float(np.sqrt(variance)) if variance is not None else None,
+        "minimum": float(finite.min()),
+        "maximum": float(finite.max()),
+    }
+
+
+def numeric_state(
+    values: pd.Series | np.ndarray | list[Any],
+    *,
+    backend: BackendName | str | None = None,
+) -> dict[str, Any]:
+    """Calculate exact first/second-moment state through the selected backend."""
+    array = _float64_array(values)
+    selected = resolve_numeric_backend(backend)
+    if selected == "rust":
+        native = import_module("framevitals._native")
+        payload = native.numeric_state_f64(array)
+        return dict(payload)
+    return _numpy_numeric_state(array)
+
+
+def numeric_profile(
+    values: pd.Series | np.ndarray | list[Any],
+    *,
+    backend: BackendName | str | None = None,
+    stream_id: int = 0,
+) -> dict[str, Any]:
+    """Run the fused native profile when available.
+
+    The NumPy fallback intentionally returns only exact state for now rather
+    than pretending to reproduce native approximate sketches with a different
+    algorithm. ``sketches_available`` makes that distinction explicit.
+    """
+    array = _float64_array(values)
+    selected = resolve_numeric_backend(backend)
+    if selected == "rust":
+        native = import_module("framevitals._native")
+        payload = dict(native.numeric_profile_f64(array, stream_id=int(stream_id)))
+        payload["sketches_available"] = True
+        return payload
+
+    payload = _numpy_numeric_state(array)
+    payload["sketches_available"] = False
+    return payload
+
+
+def backend_status() -> dict[str, Any]:
+    """Return lightweight backend availability without importing native code."""
+    selected = resolve_numeric_backend("auto")
+    return {
+        "selected": selected,
+        "native_available": native_available(),
+        "environment_override": os.getenv("FRAMEVITALS_BACKEND"),
+        "eligible": ["numpy", *( ["rust"] if native_available() else [] )],
+    }

From 0c48245714aedc7239747a30b3f8d99248755218 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:15:50 +0530
Subject: [PATCH 147/439] build(native): rely on maturin PyO3 extension
 configuration

---
 rust/framevitals-py/Cargo.toml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/rust/framevitals-py/Cargo.toml b/rust/framevitals-py/Cargo.toml
index 023eb27..cec8855 100644
--- a/rust/framevitals-py/Cargo.toml
+++ b/rust/framevitals-py/Cargo.toml
@@ -11,10 +11,6 @@ name = "_native"
 path = "src/lib.rs"
 crate-type = ["cdylib", "rlib"]
 
-[features]
-default = []
-extension-module = ["pyo3/extension-module"]
-
 [dependencies]
 framevitals-core = { path = "../framevitals-core" }
 pyo3 = { version = "0.29", features = ["abi3-py311"] }

From c6f749f3d4b6a7df28a6006b879cd13a77bf5ee0 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:16:09 +0530
Subject: [PATCH 148/439] build(native): configure mixed Python Rust module
 path

---
 pyproject.toml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index 5e35283..b3905c8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -95,6 +95,10 @@ package-dir = {"" = "src"}
 where = ["src"]
 include = ["framevitals*"]
 
+[tool.maturin]
+python-source = "src"
+module-name = "framevitals._native"
+
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 addopts = "-ra"

From 03635ec08bd7c2d7ad252bc43b80bc74808ecb8c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:16:34 +0530
Subject: [PATCH 149/439] feat: route mergeable numeric state through native
 backend

---
 src/framevitals/analysis_state.py | 51 +++++++++++++++----------------
 1 file changed, 24 insertions(+), 27 deletions(-)

diff --git a/src/framevitals/analysis_state.py b/src/framevitals/analysis_state.py
index 890e2ea..6722faa 100644
--- a/src/framevitals/analysis_state.py
+++ b/src/framevitals/analysis_state.py
@@ -1,8 +1,8 @@
 """Mergeable analysis-state primitives.
 
-These classes are the reference semantics for FrameVitals' future Rust/Arrow
-streaming engine. A partition can be summarized independently and merged with
-another partition without concatenating raw rows.
+These classes are the reference semantics for FrameVitals' Rust/Arrow streaming
+engine. A partition can be summarized independently and merged with another
+partition without concatenating raw rows.
 """
 
 from __future__ import annotations
@@ -14,6 +14,8 @@
 import numpy as np
 import pandas as pd
 
+from framevitals.backends import numeric_state
+
 
 @dataclass(slots=True)
 class NumericColumnState:
@@ -29,26 +31,25 @@ class NumericColumnState:
 
     @classmethod
     def from_series(cls, series: pd.Series) -> "NumericColumnState":
-        values = pd.to_numeric(series, errors="coerce").to_numpy(
-            dtype="float64",
-            na_value=np.nan,
-        )
-        missing = int(np.isnan(values).sum())
-        infinite = int(np.isinf(values).sum())
-        finite = values[np.isfinite(values)]
-        if finite.size == 0:
-            return cls(missing=missing, infinite=infinite)
-
-        mean = float(finite.mean())
-        centered = finite - mean
+        payload = numeric_state(series)
+        count = int(payload["count"])
+        variance = payload.get("variance")
         return cls(
-            count=int(finite.size),
-            missing=missing,
-            mean=mean,
-            m2=float(np.dot(centered, centered)),
-            minimum=float(finite.min()),
-            maximum=float(finite.max()),
-            infinite=infinite,
+            count=count,
+            missing=int(payload["missing"]),
+            mean=float(payload["mean"]) if count else 0.0,
+            m2=(float(variance) * (count - 1) if variance is not None and count >= 2 else 0.0),
+            minimum=(
+                float(payload["minimum"])
+                if payload.get("minimum") is not None
+                else None
+            ),
+            maximum=(
+                float(payload["maximum"])
+                if payload.get("maximum") is not None
+                else None
+            ),
+            infinite=int(payload["infinite"]),
         )
 
     def merge(self, other: "NumericColumnState") -> "NumericColumnState":
@@ -77,11 +78,7 @@ def merge(self, other: "NumericColumnState") -> "NumericColumnState":
         total = self.count + other.count
         delta = other.mean - self.mean
         mean = self.mean + delta * other.count / total
-        m2 = (
-            self.m2
-            + other.m2
-            + delta * delta * self.count * other.count / total
-        )
+        m2 = self.m2 + other.m2 + delta * delta * self.count * other.count / total
         minimum = min(
             value for value in (self.minimum, other.minimum) if value is not None
         )

From 58c428fb7aca1b967b050e2b514d714a70856cd9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:16:49 +0530
Subject: [PATCH 150/439] test: cover numeric backend routing

---
 tests/test_backends.py | 68 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 tests/test_backends.py

diff --git a/tests/test_backends.py b/tests/test_backends.py
new file mode 100644
index 0000000..533db46
--- /dev/null
+++ b/tests/test_backends.py
@@ -0,0 +1,68 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from framevitals import backends
+from framevitals.analysis_state import NumericColumnState
+
+
+def test_numpy_numeric_state_matches_reference_semantics():
+    payload = backends.numeric_state(
+        np.array([1.0, 2.0, np.nan, np.inf, 4.0], dtype=np.float64),
+        backend="numpy",
+    )
+
+    assert payload["backend"] == "numpy"
+    assert payload["observations"] == 5
+    assert payload["count"] == 3
+    assert payload["missing"] == 1
+    assert payload["infinite"] == 1
+    assert payload["mean"] == pytest.approx(7.0 / 3.0)
+    assert payload["variance"] == pytest.approx(7.0 / 3.0)
+    assert payload["minimum"] == 1.0
+    assert payload["maximum"] == 4.0
+
+
+def test_auto_backend_falls_back_without_native_extension(monkeypatch):
+    monkeypatch.setattr(backends, "native_available", lambda: False)
+    assert backends.resolve_numeric_backend("auto") == "numpy"
+
+
+def test_explicit_rust_backend_requires_native_extension(monkeypatch):
+    monkeypatch.setattr(backends, "native_available", lambda: False)
+    with pytest.raises(RuntimeError, match="Rust backend was requested"):
+        backends.resolve_numeric_backend("rust")
+
+
+def test_backend_environment_override(monkeypatch):
+    monkeypatch.setenv("FRAMEVITALS_BACKEND", "numpy")
+    assert backends.resolve_numeric_backend() == "numpy"
+
+
+def test_numeric_profile_numpy_discloses_missing_sketch_layer():
+    payload = backends.numeric_profile([1.0, 2.0, 3.0], backend="numpy")
+    assert payload["backend"] == "numpy"
+    assert payload["sketches_available"] is False
+
+
+def test_analysis_state_reconstructs_m2_from_backend_variance(monkeypatch):
+    monkeypatch.setattr(
+        "framevitals.analysis_state.numeric_state",
+        lambda series: {
+            "count": 3,
+            "missing": 1,
+            "infinite": 0,
+            "mean": 2.0,
+            "variance": 1.0,
+            "std": 1.0,
+            "minimum": 1.0,
+            "maximum": 3.0,
+        },
+    )
+    state = NumericColumnState.from_series(pd.Series([1.0, 2.0, 3.0, None]))
+
+    assert state.count == 3
+    assert state.missing == 1
+    assert state.mean == 2.0
+    assert state.m2 == 2.0
+    assert state.variance == 1.0

From 65774d22044a05c7bb6d42b7ec79a7a98f40f019 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:17:18 +0530
Subject: [PATCH 151/439] ci: build and smoke test native Python bridge

---
 .github/workflows/test.yml | 44 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 43 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e09b6ab..2940899 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -87,7 +87,7 @@ jobs:
         run: pytest -v
 
   native-core:
-    name: native core (Rust)
+    name: native core (Rust + Python bridge)
     runs-on: ubuntu-latest
 
     steps:
@@ -96,6 +96,11 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
       - name: Configure Rust tooling
         run: |
           rustup default stable
@@ -112,6 +117,43 @@ jobs:
       - name: Lint Rust workspace
         run: cargo clippy --manifest-path rust/Cargo.toml --workspace --all-targets --all-features -- -D warnings
 
+      - name: Install native build tooling
+        run: |
+          python -m pip install --upgrade pip
+          pip install "maturin>=1.14,<2" "numpy>=1.26"
+
+      - name: Build native Python wheel
+        run: |
+          maturin build \
+            --release \
+            --manifest-path rust/framevitals-py/Cargo.toml \
+            --interpreter python \
+            --out dist-native
+
+      - name: Verify native Python bridge
+        run: |
+          pip install --force-reinstall --no-deps dist-native/*.whl
+          python - <<'PY'
+          import numpy as np
+          from framevitals import _native
+
+          values = np.ascontiguousarray(
+              np.array([1.0, 2.0, np.nan, np.inf, 4.0, 4.0], dtype=np.float64)
+          )
+          state = _native.numeric_state_f64(values)
+          assert state["backend"] == "rust"
+          assert state["count"] == 4
+          assert state["missing"] == 1
+          assert state["infinite"] == 1
+          assert abs(state["mean"] - 2.75) < 1e-12
+
+          profile = _native.numeric_profile_f64(values, stream_id=17)
+          assert profile["cardinality_estimate"] >= 3
+          assert profile["quantiles"]["p50"] is not None
+          assert profile["reservoir"]
+          print(_native.backend_info())
+          PY
+
   optional-features:
     name: optional features (Python 3.12)
     runs-on: ubuntu-latest

From 13b092c676a6002d2cac966fc60062c9594f49e9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:18:35 +0530
Subject: [PATCH 152/439] style(native): apply rustfmt to sketch layer

---
 rust/framevitals-core/src/sketches.rs | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/rust/framevitals-core/src/sketches.rs b/rust/framevitals-core/src/sketches.rs
index 8595c0a..53e7a51 100644
--- a/rust/framevitals-core/src/sketches.rs
+++ b/rust/framevitals-core/src/sketches.rs
@@ -34,7 +34,10 @@ pub struct HyperLogLog {
 
 impl HyperLogLog {
     pub fn new(precision: u8) -> Self {
-        assert!((4..=16).contains(&precision), "HLL precision must be 4..=16");
+        assert!(
+            (4..=16).contains(&precision),
+            "HLL precision must be 4..=16"
+        );
         Self {
             precision,
             registers: vec![0; 1usize << precision],
@@ -273,7 +276,10 @@ impl HeavyHittersSketch {
 
     #[must_use]
     pub fn merge(mut self, other: Self) -> Self {
-        assert_eq!(self.capacity, other.capacity, "heavy-hitter capacity mismatch");
+        assert_eq!(
+            self.capacity, other.capacity,
+            "heavy-hitter capacity mismatch"
+        );
         for (key, count) in other.counters {
             self.observe_weighted(key, count);
         }
@@ -423,8 +429,7 @@ impl NumericSketchState {
 #[cfg(test)]
 mod tests {
     use super::{
-        HeavyHittersSketch, HyperLogLog, LogQuantileSketch, NumericSketchState,
-        PriorityReservoir,
+        HeavyHittersSketch, HyperLogLog, LogQuantileSketch, NumericSketchState, PriorityReservoir,
     };
 
     #[test]

From e07cd320302ffb8a49d3091e3b749add431fd1d1 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:18:56 +0530
Subject: [PATCH 153/439] style(native): apply rustfmt to PyO3 bridge

---
 rust/framevitals-py/src/lib.rs | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/rust/framevitals-py/src/lib.rs b/rust/framevitals-py/src/lib.rs
index a28e194..1005f0f 100644
--- a/rust/framevitals-py/src/lib.rs
+++ b/rust/framevitals-py/src/lib.rs
@@ -7,17 +7,18 @@ use pyo3::exceptions::PyBufferError;
 use pyo3::prelude::*;
 use pyo3::types::PyDict;
 
-fn exact_state_dict<'py>(
-    py: Python<'py>,
-    state: &NumericState,
-) -> PyResult<Bound<'py, PyDict>> {
+fn exact_state_dict<'py>(py: Python<'py>, state: &NumericState) -> PyResult<Bound<'py, PyDict>> {
     let payload = PyDict::new(py);
     payload.set_item("count", state.count)?;
     payload.set_item("missing", state.missing)?;
     payload.set_item("infinite", state.infinite)?;
     payload.set_item(
         "mean",
-        if state.count > 0 { Some(state.mean) } else { None },
+        if state.count > 0 {
+            Some(state.mean)
+        } else {
+            None
+        },
     )?;
     payload.set_item("variance", state.variance())?;
     payload.set_item("std", state.standard_deviation())?;
@@ -103,10 +104,7 @@ fn numeric_profile_f64(
     ] {
         quantiles.set_item(name, sketches.quantiles.quantile(q))?;
     }
-    quantiles.set_item(
-        "relative_accuracy",
-        sketches.quantiles.relative_accuracy(),
-    )?;
+    quantiles.set_item("relative_accuracy", sketches.quantiles.relative_accuracy())?;
     payload.set_item("quantiles", quantiles)?;
     payload.set_item("heavy_hitters", sketches.heavy_hitters.candidates())?;
     payload.set_item("reservoir", sketches.reservoir.values())?;

From cb376a28e25dcae7d5e822928bf5c06c0cc69469 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:22:04 +0530
Subject: [PATCH 154/439] feat: use native fused numeric profiling for large
 datasets

---
 src/framevitals/profiler.py | 147 +++++++++++++++++++++++++++++++-----
 1 file changed, 129 insertions(+), 18 deletions(-)

diff --git a/src/framevitals/profiler.py b/src/framevitals/profiler.py
index 4f7a79a..a5e09e1 100644
--- a/src/framevitals/profiler.py
+++ b/src/framevitals/profiler.py
@@ -1,11 +1,17 @@
 from __future__ import annotations
 
+import os
+
 import numpy as np
 import pandas as pd
 
+from framevitals.backends import numeric_profile, resolve_numeric_backend
+
 MAX_CORRELATION_COLUMNS = 100
 MAX_EXACT_DUPLICATE_CELLS = 50_000_000
 DUPLICATE_SAMPLE_ROWS = 50_000
+NATIVE_NUMERIC_PROFILE_MIN_ROWS = 50_000
+NATIVE_NUMERIC_PROFILE_MIN_CELLS = 1_000_000
 
 
 def clean_value(value):
@@ -42,10 +48,18 @@ def detect_column_types(df):
     return numeric_cols, categorical_cols, date_cols
 
 
-def _missing_counts(df: pd.DataFrame) -> pd.Series:
+def _missing_counts(
+    df: pd.DataFrame,
+    *,
+    precomputed: dict[str, int] | None = None,
+) -> pd.Series:
     """Count missing values without materializing a frame-sized boolean table."""
+    known = precomputed or {}
     return pd.Series(
-        {column: int(df[column].isna().sum()) for column in df.columns},
+        {
+            column: int(known[column]) if column in known else int(df[column].isna().sum())
+            for column in df.columns
+        },
         dtype="int64",
     )
 
@@ -92,9 +106,6 @@ def _bounded_correlations(
     selected = numeric_cols
     truncated = len(selected) > MAX_CORRELATION_COLUMNS
     if truncated:
-        # Prefer columns with the most usable observations. This is a safe
-        # compatibility fallback; the sparse relationship graph will replace
-        # dense truncation for ultra-wide datasets.
         non_missing = {
             column: int(df[column].notna().sum())
             for column in numeric_cols
@@ -120,25 +131,123 @@ def _bounded_correlations(
     }
 
 
+def _round_optional(value, digits: int = 3):
+    if value is None:
+        return None
+    return round(float(value), digits)
+
+
+def _pandas_numeric_summary(
+    df: pd.DataFrame,
+    numeric_cols: list[str],
+) -> tuple[dict, dict, dict[str, int]]:
+    if not numeric_cols:
+        return {}, {
+            "backend": "not_applicable",
+            "method": "not_applicable",
+            "approximate_quantiles": False,
+        }, {}
+
+    summary = (
+        df[numeric_cols]
+        .describe()
+        .T
+        .replace({np.nan: None})
+        .round(3)
+        .to_dict(orient="index")
+    )
+    return summary, {
+        "backend": "pandas",
+        "method": "pandas_describe",
+        "approximate_quantiles": False,
+        "columns_profiled": len(numeric_cols),
+    }, {}
+
+
+def _native_numeric_summary(
+    df: pd.DataFrame,
+    numeric_cols: list[str],
+) -> tuple[dict, dict, dict[str, int]]:
+    summary: dict[str, dict] = {}
+    missing: dict[str, int] = {}
+    relative_accuracy = None
+
+    for stream_id, column in enumerate(numeric_cols):
+        payload = numeric_profile(df[column], backend="rust", stream_id=stream_id)
+        quantiles = payload.get("quantiles", {})
+        relative_accuracy = quantiles.get("relative_accuracy", relative_accuracy)
+        missing[column] = int(payload["missing"])
+        summary[column] = {
+            "count": int(payload["count"]),
+            "mean": _round_optional(payload.get("mean")),
+            "std": _round_optional(payload.get("std")),
+            "min": _round_optional(payload.get("minimum")),
+            "25%": _round_optional(quantiles.get("p25")),
+            "50%": _round_optional(quantiles.get("p50")),
+            "75%": _round_optional(quantiles.get("p75")),
+            "max": _round_optional(payload.get("maximum")),
+        }
+
+    return summary, {
+        "backend": "rust",
+        "method": "native_fused_numeric_column_scan",
+        "approximate_quantiles": True,
+        "quantile_relative_accuracy": relative_accuracy,
+        "finite_only_moments": True,
+        "columns_profiled": len(numeric_cols),
+        "raw_observations_retained": False,
+    }, missing
+
+
+def _numeric_summary(
+    df: pd.DataFrame,
+    numeric_cols: list[str],
+) -> tuple[dict, dict, dict[str, int]]:
+    if not numeric_cols:
+        return _pandas_numeric_summary(df, numeric_cols)
+
+    requested = os.getenv("FRAMEVITALS_BACKEND", "auto").strip().lower()
+    selected = resolve_numeric_backend()
+    numeric_cells = int(len(df)) * len(numeric_cols)
+    native_worthwhile = (
+        len(df) >= NATIVE_NUMERIC_PROFILE_MIN_ROWS
+        or numeric_cells >= NATIVE_NUMERIC_PROFILE_MIN_CELLS
+    )
+    use_native = selected == "rust" and (requested == "rust" or native_worthwhile)
+
+    if not use_native:
+        summary, metadata, missing = _pandas_numeric_summary(df, numeric_cols)
+        metadata["native_eligible"] = selected == "rust"
+        metadata["native_threshold_reached"] = native_worthwhile
+        return summary, metadata, missing
+
+    try:
+        return _native_numeric_summary(df, numeric_cols)
+    except Exception as exc:
+        if requested == "rust":
+            raise
+        summary, metadata, missing = _pandas_numeric_summary(df, numeric_cols)
+        metadata.update({
+            "native_eligible": True,
+            "native_threshold_reached": True,
+            "fallback_from": "rust",
+            "fallback_reason": f"{type(exc).__name__}: {exc}",
+        })
+        return summary, metadata, missing
+
+
 def build_profile(df):
     rows, columns = df.shape
     numeric_cols, categorical_cols, date_cols = detect_column_types(df)
 
-    missing_counts = _missing_counts(df)
+    numeric_summary, numeric_summary_metadata, numeric_missing = _numeric_summary(
+        df,
+        numeric_cols,
+    )
+    missing_counts = _missing_counts(df, precomputed=numeric_missing)
     missing_percent = (missing_counts / max(rows, 1) * 100).round(2)
     duplicate_rows, duplicate_metadata = _duplicate_profile(df)
 
-    numeric_summary = {}
-    if numeric_cols:
-        numeric_summary = (
-            df[numeric_cols]
-            .describe()
-            .T
-            .replace({np.nan: None})
-            .round(3)
-            .to_dict(orient="index")
-        )
-
     categorical_summary = {}
     for col in categorical_cols:
         counts = df[col].value_counts(dropna=False).head(10)
@@ -149,7 +258,8 @@ def build_profile(df):
 
     correlations, correlation_metadata = _bounded_correlations(df, numeric_cols)
 
-    preview = df.head(15).where(df.head(15).notna(), None).to_dict(orient="records")
+    preview_frame = df.head(15)
+    preview = preview_frame.where(preview_frame.notna(), None).to_dict(orient="records")
 
     return {
         "shape": {"rows": rows, "columns": columns},
@@ -165,6 +275,7 @@ def build_profile(df):
         "duplicate_metadata": duplicate_metadata,
         "memory_usage_mb": round(df.memory_usage(deep=True).sum() / (1024 * 1024), 3),
         "numeric_summary": numeric_summary,
+        "numeric_summary_metadata": numeric_summary_metadata,
         "categorical_summary": categorical_summary,
         "correlations": correlations,
         "correlation_metadata": correlation_metadata,

From 1cee766de8b423c31d5a120f4c328c0d6e0068b2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:22:27 +0530
Subject: [PATCH 155/439] test: cover native profiler routing and fallback

---
 tests/test_native_profiler_routing.py | 120 ++++++++++++++++++++++++++
 1 file changed, 120 insertions(+)
 create mode 100644 tests/test_native_profiler_routing.py

diff --git a/tests/test_native_profiler_routing.py b/tests/test_native_profiler_routing.py
new file mode 100644
index 0000000..99ef338
--- /dev/null
+++ b/tests/test_native_profiler_routing.py
@@ -0,0 +1,120 @@
+import pandas as pd
+import pytest
+
+import framevitals.profiler as profiler
+
+
+def _frame() -> pd.DataFrame:
+    return pd.DataFrame({
+        "value": [1.0, 2.0, None, 4.0],
+        "other": [10.0, 20.0, 30.0, 40.0],
+        "label": ["a", "b", "a", "b"],
+    })
+
+
+def test_small_profile_keeps_exact_pandas_summary_when_native_is_available(monkeypatch):
+    monkeypatch.delenv("FRAMEVITALS_BACKEND", raising=False)
+    monkeypatch.setattr(profiler, "resolve_numeric_backend", lambda: "rust")
+
+    result = profiler.build_profile(_frame())
+
+    metadata = result["numeric_summary_metadata"]
+    assert metadata["backend"] == "pandas"
+    assert metadata["approximate_quantiles"] is False
+    assert metadata["native_eligible"] is True
+    assert metadata["native_threshold_reached"] is False
+    assert result["numeric_summary"]["value"]["50%"] == pytest.approx(2.0)
+
+
+def test_forced_native_profile_reuses_missing_counts_and_preserves_summary_shape(monkeypatch):
+    monkeypatch.setenv("FRAMEVITALS_BACKEND", "rust")
+    monkeypatch.setattr(profiler, "resolve_numeric_backend", lambda: "rust")
+
+    calls = []
+
+    def fake_numeric_profile(series, *, backend, stream_id):
+        calls.append((series.name, backend, stream_id))
+        if series.name == "value":
+            return {
+                "count": 3,
+                "missing": 1,
+                "infinite": 0,
+                "mean": 7.0 / 3.0,
+                "std": 1.527525,
+                "minimum": 1.0,
+                "maximum": 4.0,
+                "quantiles": {
+                    "p25": 1.5,
+                    "p50": 2.0,
+                    "p75": 3.0,
+                    "relative_accuracy": 0.01,
+                },
+            }
+        return {
+            "count": 4,
+            "missing": 0,
+            "infinite": 0,
+            "mean": 25.0,
+            "std": 12.909944,
+            "minimum": 10.0,
+            "maximum": 40.0,
+            "quantiles": {
+                "p25": 17.5,
+                "p50": 25.0,
+                "p75": 32.5,
+                "relative_accuracy": 0.01,
+            },
+        }
+
+    monkeypatch.setattr(profiler, "numeric_profile", fake_numeric_profile)
+    result = profiler.build_profile(_frame())
+
+    assert calls == [("value", "rust", 0), ("other", "rust", 1)]
+    assert result["missing_counts"]["value"] == 1
+    assert result["missing_counts"]["other"] == 0
+    assert set(result["numeric_summary"]["value"]) == {
+        "count",
+        "mean",
+        "std",
+        "min",
+        "25%",
+        "50%",
+        "75%",
+        "max",
+    }
+    metadata = result["numeric_summary_metadata"]
+    assert metadata["backend"] == "rust"
+    assert metadata["approximate_quantiles"] is True
+    assert metadata["quantile_relative_accuracy"] == 0.01
+    assert metadata["raw_observations_retained"] is False
+
+
+def test_auto_native_failure_falls_back_and_discloses_reason(monkeypatch):
+    monkeypatch.delenv("FRAMEVITALS_BACKEND", raising=False)
+    monkeypatch.setattr(profiler, "resolve_numeric_backend", lambda: "rust")
+    monkeypatch.setattr(profiler, "NATIVE_NUMERIC_PROFILE_MIN_ROWS", 1)
+    monkeypatch.setattr(
+        profiler,
+        "numeric_profile",
+        lambda *args, **kwargs: (_ for _ in ()).throw(BufferError("not contiguous")),
+    )
+
+    result = profiler.build_profile(_frame())
+    metadata = result["numeric_summary_metadata"]
+
+    assert metadata["backend"] == "pandas"
+    assert metadata["fallback_from"] == "rust"
+    assert "BufferError" in metadata["fallback_reason"]
+
+
+def test_forced_native_failure_is_not_silently_hidden(monkeypatch):
+    monkeypatch.setenv("FRAMEVITALS_BACKEND", "rust")
+    monkeypatch.setattr(profiler, "resolve_numeric_backend", lambda: "rust")
+    monkeypatch.setattr(
+        profiler,
+        "numeric_profile",
+        lambda *args, **kwargs: (_ for _ in ()).throw(BufferError("bad buffer")),
+    )
+
+    with pytest.raises(BufferError, match="bad buffer"):
+        profiler.build_profile(_frame())

From eff350dd0fe87a54b4eb3df80883d81c1c05e2c7 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:22:50 +0530
Subject: [PATCH 156/439] ci: verify public profiler uses native backend

---
 .github/workflows/test.yml | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2940899..1cefe34 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -120,7 +120,7 @@ jobs:
       - name: Install native build tooling
         run: |
           python -m pip install --upgrade pip
-          pip install "maturin>=1.14,<2" "numpy>=1.26"
+          pip install "maturin>=1.14,<2" "numpy>=1.26" "pandas>=2.2"
 
       - name: Build native Python wheel
         run: |
@@ -130,12 +130,14 @@ jobs:
             --interpreter python \
             --out dist-native
 
-      - name: Verify native Python bridge
+      - name: Verify native Python bridge and profiler routing
         run: |
           pip install --force-reinstall --no-deps dist-native/*.whl
-          python - <<'PY'
+          FRAMEVITALS_BACKEND=rust python - <<'PY'
           import numpy as np
+          import pandas as pd
           from framevitals import _native
+          from framevitals.profiler import build_profile
 
           values = np.ascontiguousarray(
               np.array([1.0, 2.0, np.nan, np.inf, 4.0, 4.0], dtype=np.float64)
@@ -147,10 +149,19 @@ jobs:
           assert state["infinite"] == 1
           assert abs(state["mean"] - 2.75) < 1e-12
 
-          profile = _native.numeric_profile_f64(values, stream_id=17)
-          assert profile["cardinality_estimate"] >= 3
-          assert profile["quantiles"]["p50"] is not None
-          assert profile["reservoir"]
+          native_profile = _native.numeric_profile_f64(values, stream_id=17)
+          assert native_profile["cardinality_estimate"] >= 3
+          assert native_profile["quantiles"]["p50"] is not None
+          assert native_profile["reservoir"]
+
+          public_profile = build_profile(pd.DataFrame({
+              "value": [1.0, 2.0, None, 4.0, 4.0],
+              "other": [10.0, 20.0, 30.0, 40.0, 50.0],
+          }))
+          assert public_profile["numeric_summary_metadata"]["backend"] == "rust"
+          assert public_profile["numeric_summary_metadata"]["approximate_quantiles"] is True
+          assert public_profile["missing_counts"]["value"] == 1
+          assert public_profile["numeric_summary"]["value"]["count"] == 4
           print(_native.backend_info())
           PY
 

From 459e8709f98ee9a910cb6b50f3f3a8074782d094 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:24:50 +0530
Subject: [PATCH 157/439] feat(native): add persistent numeric accumulator

---
 rust/framevitals-py/src/lib.rs | 158 +++++++++++++++++++++++++--------
 1 file changed, 120 insertions(+), 38 deletions(-)

diff --git a/rust/framevitals-py/src/lib.rs b/rust/framevitals-py/src/lib.rs
index 1005f0f..9baec24 100644
--- a/rust/framevitals-py/src/lib.rs
+++ b/rust/framevitals-py/src/lib.rs
@@ -27,6 +27,39 @@ fn exact_state_dict<'py>(py: Python<'py>, state: &NumericState) -> PyResult<Boun
     Ok(payload)
 }
 
+fn profile_dict<'py>(
+    py: Python<'py>,
+    state: &NumericState,
+    sketches: &NumericSketchState,
+    observations: u64,
+) -> PyResult<Bound<'py, PyDict>> {
+    let payload = exact_state_dict(py, state)?;
+    payload.set_item("backend", "rust")?;
+    payload.set_item("observations", observations)?;
+    payload.set_item(
+        "cardinality_estimate",
+        sketches.cardinality.estimate().round() as u64,
+    )?;
+
+    let quantiles = PyDict::new(py);
+    for (name, q) in [
+        ("p01", 0.01),
+        ("p05", 0.05),
+        ("p25", 0.25),
+        ("p50", 0.50),
+        ("p75", 0.75),
+        ("p95", 0.95),
+        ("p99", 0.99),
+    ] {
+        quantiles.set_item(name, sketches.quantiles.quantile(q))?;
+    }
+    quantiles.set_item("relative_accuracy", sketches.quantiles.relative_accuracy())?;
+    payload.set_item("quantiles", quantiles)?;
+    payload.set_item("heavy_hitters", sketches.heavy_hitters.candidates())?;
+    payload.set_item("reservoir", sketches.reservoir.values())?;
+    Ok(payload)
+}
+
 fn checked_buffer<'py>(values: &Bound<'py, PyAny>) -> PyResult<PyBuffer<f64>> {
     let buffer = PyBuffer::<f64>::get(values)?;
     if buffer.dimensions() != 1 {
@@ -42,6 +75,83 @@ fn checked_buffer<'py>(values: &Bound<'py, PyAny>) -> PyResult<PyBuffer<f64>> {
     Ok(buffer)
 }
 
+fn update_states(
+    py: Python<'_>,
+    values: &Bound<'_, PyAny>,
+    state: &mut NumericState,
+    sketches: &mut NumericSketchState,
+    stream_id: u64,
+    sequence: &mut u64,
+) -> PyResult<u64> {
+    let buffer = checked_buffer(values)?;
+    let slice = buffer.as_slice(py).ok_or_else(|| {
+        PyBufferError::new_err(
+            "FrameVitals could not borrow the supplied float64 buffer as a contiguous slice.",
+        )
+    })?;
+
+    for value in slice {
+        let value = value.get();
+        state.observe(Some(value));
+        sketches.observe(value, stream_id, *sequence);
+        *sequence = sequence.wrapping_add(1);
+    }
+    Ok(buffer.item_count() as u64)
+}
+
+#[pyclass]
+struct NumericAccumulator {
+    state: NumericState,
+    sketches: NumericSketchState,
+    stream_id: u64,
+    sequence: u64,
+    observations: u64,
+}
+
+#[pymethods]
+impl NumericAccumulator {
+    #[new]
+    #[pyo3(signature = (stream_id = 0))]
+    fn new(stream_id: u64) -> Self {
+        Self {
+            state: NumericState::default(),
+            sketches: NumericSketchState::default(),
+            stream_id,
+            sequence: 0,
+            observations: 0,
+        }
+    }
+
+    fn update_f64(&mut self, py: Python<'_>, values: &Bound<'_, PyAny>) -> PyResult<()> {
+        let observed = update_states(
+            py,
+            values,
+            &mut self.state,
+            &mut self.sketches,
+            self.stream_id,
+            &mut self.sequence,
+        )?;
+        self.observations = self.observations.wrapping_add(observed);
+        Ok(())
+    }
+
+    fn snapshot(&self, py: Python<'_>) -> PyResult<Py<PyDict>> {
+        Ok(profile_dict(py, &self.state, &self.sketches, self.observations)?.unbind())
+    }
+
+    fn reset(&mut self) {
+        self.state = NumericState::default();
+        self.sketches = NumericSketchState::default();
+        self.sequence = 0;
+        self.observations = 0;
+    }
+
+    #[getter]
+    fn observations(&self) -> u64 {
+        self.observations
+    }
+}
+
 #[pyfunction]
 fn numeric_state_f64(py: Python<'_>, values: &Bound<'_, PyAny>) -> PyResult<Py<PyDict>> {
     let buffer = checked_buffer(values)?;
@@ -68,47 +178,18 @@ fn numeric_profile_f64(
     values: &Bound<'_, PyAny>,
     stream_id: u64,
 ) -> PyResult<Py<PyDict>> {
-    let buffer = checked_buffer(values)?;
-    let slice = buffer.as_slice(py).ok_or_else(|| {
-        PyBufferError::new_err(
-            "FrameVitals could not borrow the supplied float64 buffer as a contiguous slice.",
-        )
-    })?;
-
     let mut state = NumericState::default();
     let mut sketches = NumericSketchState::default();
-
-    for (index, value) in slice.iter().enumerate() {
-        let value = value.get();
-        state.observe(Some(value));
-        sketches.observe(value, stream_id, index as u64);
-    }
-
-    let payload = exact_state_dict(py, &state)?;
-    payload.set_item("backend", "rust")?;
-    payload.set_item("observations", buffer.item_count())?;
-    payload.set_item(
-        "cardinality_estimate",
-        sketches.cardinality.estimate().round() as u64,
+    let mut sequence = 0_u64;
+    let observations = update_states(
+        py,
+        values,
+        &mut state,
+        &mut sketches,
+        stream_id,
+        &mut sequence,
     )?;
-
-    let quantiles = PyDict::new(py);
-    for (name, q) in [
-        ("p01", 0.01),
-        ("p05", 0.05),
-        ("p25", 0.25),
-        ("p50", 0.50),
-        ("p75", 0.75),
-        ("p95", 0.95),
-        ("p99", 0.99),
-    ] {
-        quantiles.set_item(name, sketches.quantiles.quantile(q))?;
-    }
-    quantiles.set_item("relative_accuracy", sketches.quantiles.relative_accuracy())?;
-    payload.set_item("quantiles", quantiles)?;
-    payload.set_item("heavy_hitters", sketches.heavy_hitters.candidates())?;
-    payload.set_item("reservoir", sketches.reservoir.values())?;
-    Ok(payload.unbind())
+    Ok(profile_dict(py, &state, &sketches, observations)?.unbind())
 }
 
 #[pyfunction]
@@ -118,6 +199,7 @@ fn backend_info() -> (&'static str, &'static str) {
 
 #[pymodule]
 fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> {
+    module.add_class::<NumericAccumulator>()?;
     module.add_function(wrap_pyfunction!(numeric_state_f64, module)?)?;
     module.add_function(wrap_pyfunction!(numeric_profile_f64, module)?)?;
     module.add_function(wrap_pyfunction!(backend_info, module)?)?;

From acde75bfa683b400c0f78d4e0e30040203fc0a27 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:25:18 +0530
Subject: [PATCH 158/439] build: add optional Arrow scale capability

---
 pyproject.toml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index b3905c8..cc02a7a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -60,6 +60,9 @@ web = [
     "gunicorn>=22.0",
     "werkzeug>=3.0",
 ]
+arrow = [
+    "pyarrow>=25.0,<26",
+]
 all = [
     "xgboost>=2.0",
     "lightgbm>=4.3",

From 990786a035a838b866fb05d030ea8df8aaef69d5 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:25:38 +0530
Subject: [PATCH 159/439] feat: add optional streaming Parquet source

---
 src/framevitals/sources.py | 104 ++++++++++++++++++++++++++++++++-----
 1 file changed, 90 insertions(+), 14 deletions(-)

diff --git a/src/framevitals/sources.py b/src/framevitals/sources.py
index 092486a..667cc3d 100644
--- a/src/framevitals/sources.py
+++ b/src/framevitals/sources.py
@@ -1,15 +1,16 @@
-"""Dataset source abstraction for FrameVitals.
+"""Dataset source abstractions for FrameVitals.
 
-Today the execution pipeline still materializes supported files through pandas.
-This module establishes the source contract needed for future Arrow/DataFusion
-streaming without changing the public ``fv.analyze(data)`` API.
+Sources expose cheap metadata before analysis. Streaming-capable sources can
+additionally yield bounded record batches, allowing focused operations to avoid
+materializing the complete dataset in pandas.
 """
 
 from __future__ import annotations
 
+from collections.abc import Iterator, Sequence
 from dataclasses import asdict, dataclass
 from pathlib import Path
-from typing import Protocol, runtime_checkable
+from typing import Any, Protocol, runtime_checkable
 
 import pandas as pd
 
@@ -43,6 +44,18 @@ def inspect(self) -> DatasetMetadata: ...
     def load(self) -> pd.DataFrame: ...
 
 
+@runtime_checkable
+class StreamingDatasetSource(DatasetSource, Protocol):
+    """Optional extension for sources that can yield bounded record batches."""
+
+    def iter_batches(
+        self,
+        *,
+        batch_size: int = 65_536,
+        columns: Sequence[str] | None = None,
+    ) -> Iterator[Any]: ...
+
+
 @dataclass(slots=True)
 class PandasSource:
     dataframe: pd.DataFrame
@@ -74,15 +87,8 @@ class FileSource:
     path: Path
 
     def inspect(self) -> DatasetMetadata:
-        if not self.path.exists():
-            raise FileNotFoundError(f"Dataset not found: {self.path}")
-        if not self.path.is_file():
-            raise ValueError(f"Expected a dataset file, got: {self.path}")
-
+        _validate_file_path(self.path)
         suffix = self.path.suffix.lower().lstrip(".") or "unknown"
-        # The current loader materializes every supported file. These capability
-        # flags describe what FrameVitals can do *today*, not what the format
-        # could theoretically support. Arrow/Parquet sources will override them.
         return DatasetMetadata(
             name=self.path.name,
             kind="file",
@@ -102,12 +108,82 @@ def load(self) -> pd.DataFrame:
         return dataframe
 
 
+@dataclass(slots=True)
+class ParquetSource:
+    """Projection-aware, streaming Parquet source backed by optional PyArrow."""
+
+    path: Path
+
+    def _parquet_file(self):
+        _validate_file_path(self.path)
+        try:
+            import pyarrow.parquet as pq
+        except ImportError as exc:
+            raise ImportError(
+                "Parquet streaming requires the optional Arrow capability. "
+                'Install it with: pip install "framevitals[arrow]"'
+            ) from exc
+        return pq.ParquetFile(self.path)
+
+    def inspect(self) -> DatasetMetadata:
+        parquet_file = self._parquet_file()
+        metadata = parquet_file.metadata
+        return DatasetMetadata(
+            name=self.path.name,
+            kind="file",
+            format="parquet",
+            rows=int(metadata.num_rows),
+            columns=int(metadata.num_columns),
+            size_bytes=int(self.path.stat().st_size),
+            materialized=False,
+            supports_projection=True,
+            supports_streaming=True,
+        )
+
+    def schema(self):
+        """Return the Arrow schema without reading all row data."""
+        return self._parquet_file().schema_arrow
+
+    def iter_batches(
+        self,
+        *,
+        batch_size: int = 65_536,
+        columns: Sequence[str] | None = None,
+    ) -> Iterator[Any]:
+        if batch_size < 1:
+            raise ValueError("batch_size must be at least 1.")
+        parquet_file = self._parquet_file()
+        yield from parquet_file.iter_batches(
+            batch_size=int(batch_size),
+            columns=list(columns) if columns is not None else None,
+            use_threads=True,
+        )
+
+    def load(self) -> pd.DataFrame:
+        parquet_file = self._parquet_file()
+        table = parquet_file.read(use_threads=True)
+        dataframe = table.to_pandas()
+        if dataframe.empty:
+            raise ValueError(f"Dataset is empty: {self.path}")
+        return dataframe
+
+
+def _validate_file_path(path: Path) -> None:
+    if not path.exists():
+        raise FileNotFoundError(f"Dataset not found: {path}")
+    if not path.is_file():
+        raise ValueError(f"Expected a dataset file, got: {path}")
+
+
 def resolve_source(data: str | Path | pd.DataFrame | DatasetSource) -> DatasetSource:
     """Normalize supported user inputs into a DatasetSource implementation."""
     if isinstance(data, pd.DataFrame):
         return PandasSource(data)
     if isinstance(data, (str, Path)):
-        return FileSource(Path(data))
+        path = Path(data)
+        if path.suffix.lower() == ".parquet":
+            return ParquetSource(path)
+        return FileSource(path)
     if isinstance(data, DatasetSource):
         return data
     raise TypeError("data must be a pandas DataFrame, dataset path, or DatasetSource.")

From 1c4629ed179e8a0ac6d96e8c1935088cf3d4a048 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:26:27 +0530
Subject: [PATCH 160/439] feat: expose persistent native numeric accumulator

---
 src/framevitals/backends.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/src/framevitals/backends.py b/src/framevitals/backends.py
index 3af0ecb..7398aff 100644
--- a/src/framevitals/backends.py
+++ b/src/framevitals/backends.py
@@ -136,12 +136,28 @@ def numeric_profile(
     return payload
 
 
+def create_numeric_accumulator(*, stream_id: int = 0):
+    """Create a persistent native accumulator for multi-batch scans.
+
+    Returns ``None`` when the active backend is NumPy. This keeps callers free
+    to implement a mergeable Python fallback without importing the native
+    extension unless it is actually selected.
+    """
+    if resolve_numeric_backend() != "rust":
+        return None
+    native = import_module("framevitals._native")
+    return native.NumericAccumulator(stream_id=int(stream_id))
+
+
 def backend_status() -> dict[str, Any]:
     """Return lightweight backend availability without importing native code."""
-    selected = resolve_numeric_backend("auto")
+    has_native = native_available()
+    eligible = ["numpy"]
+    if has_native:
+        eligible.append("rust")
     return {
-        "selected": selected,
-        "native_available": native_available(),
+        "selected": resolve_numeric_backend("auto"),
+        "native_available": has_native,
         "environment_override": os.getenv("FRAMEVITALS_BACKEND"),
-        "eligible": ["numpy", *( ["rust"] if native_available() else [] )],
+        "eligible": eligible,
     }

From 7b530d620e2864844a75f8d34a6bad9f0cdf3b51 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:27:14 +0530
Subject: [PATCH 161/439] feat: add bounded streaming profile for Arrow sources

---
 src/framevitals/streaming_profile.py | 395 +++++++++++++++++++++++++++
 1 file changed, 395 insertions(+)
 create mode 100644 src/framevitals/streaming_profile.py

diff --git a/src/framevitals/streaming_profile.py b/src/framevitals/streaming_profile.py
new file mode 100644
index 0000000..e9230cf
--- /dev/null
+++ b/src/framevitals/streaming_profile.py
@@ -0,0 +1,395 @@
+"""Streaming profile construction for Arrow-capable dataset sources.
+
+The streaming path keeps full-row work to mergeable column state and retains
+only a bounded, evenly spaced row sample for analyses that still require row
+relationships (categorical top values, duplicate estimation, and correlation).
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from framevitals.analysis_state import NumericColumnState
+from framevitals.backends import (
+    create_numeric_accumulator,
+    numeric_state,
+    resolve_numeric_backend,
+)
+from framevitals.profiler import _bounded_correlations, series_to_dict
+from framevitals.sources import StreamingDatasetSource
+
+
+STREAM_BATCH_SIZE = 65_536
+STREAM_SAMPLE_ROWS = 50_000
+
+
+def _require_pyarrow():
+    try:
+        import pyarrow as pa
+    except ImportError as exc:
+        raise ImportError(
+            "Streaming Arrow profiling requires the optional Arrow capability. "
+            'Install it with: pip install "framevitals[arrow]"'
+        ) from exc
+    return pa
+
+
+def _column_groups(schema) -> tuple[list[str], list[str], list[str]]:
+    pa = _require_pyarrow()
+    numeric: list[str] = []
+    categorical: list[str] = []
+    dates: list[str] = []
+
+    for field in schema:
+        dtype = field.type
+        if pa.types.is_integer(dtype) or pa.types.is_floating(dtype):
+            numeric.append(field.name)
+        elif (
+            pa.types.is_string(dtype)
+            or pa.types.is_large_string(dtype)
+            or pa.types.is_dictionary(dtype)
+            or pa.types.is_boolean(dtype)
+            or pa.types.is_binary(dtype)
+            or pa.types.is_large_binary(dtype)
+        ):
+            categorical.append(field.name)
+        elif (
+            pa.types.is_timestamp(dtype)
+            or pa.types.is_date32(dtype)
+            or pa.types.is_date64(dtype)
+            or pa.types.is_time32(dtype)
+            or pa.types.is_time64(dtype)
+            or pa.types.is_duration(dtype)
+        ):
+            dates.append(field.name)
+
+    return numeric, categorical, dates
+
+
+def _arrow_numeric_to_float64(array) -> np.ndarray:
+    """Convert one bounded Arrow numeric array to a contiguous float64 buffer."""
+    try:
+        values = array.to_numpy(zero_copy_only=False)
+        converted = np.asarray(values, dtype=np.float64)
+    except (TypeError, ValueError):
+        series = array.to_pandas()
+        converted = pd.to_numeric(series, errors="coerce").to_numpy(
+            dtype="float64",
+            na_value=np.nan,
+        )
+    return np.ascontiguousarray(converted, dtype=np.float64)
+
+
+def _state_from_payload(payload: dict[str, Any]) -> NumericColumnState:
+    count = int(payload["count"])
+    variance = payload.get("variance")
+    return NumericColumnState(
+        count=count,
+        missing=int(payload["missing"]),
+        mean=float(payload["mean"]) if count else 0.0,
+        m2=(float(variance) * (count - 1) if variance is not None and count >= 2 else 0.0),
+        minimum=(
+            float(payload["minimum"])
+            if payload.get("minimum") is not None
+            else None
+        ),
+        maximum=(
+            float(payload["maximum"])
+            if payload.get("maximum") is not None
+            else None
+        ),
+        infinite=int(payload["infinite"]),
+    )
+
+
+def _round_optional(value: Any, digits: int = 3):
+    if value is None:
+        return None
+    return round(float(value), digits)
+
+
+def _summary_from_native_snapshot(payload: dict[str, Any]) -> dict[str, Any]:
+    quantiles = payload.get("quantiles", {})
+    return {
+        "count": int(payload["count"]),
+        "mean": _round_optional(payload.get("mean")),
+        "std": _round_optional(payload.get("std")),
+        "min": _round_optional(payload.get("minimum")),
+        "25%": _round_optional(quantiles.get("p25")),
+        "50%": _round_optional(quantiles.get("p50")),
+        "75%": _round_optional(quantiles.get("p75")),
+        "max": _round_optional(payload.get("maximum")),
+    }
+
+
+def _summary_from_python_state(
+    state: NumericColumnState,
+    sample: pd.Series,
+) -> dict[str, Any]:
+    finite_sample = pd.to_numeric(sample, errors="coerce")
+    finite_sample = finite_sample[np.isfinite(finite_sample.to_numpy(dtype="float64", na_value=np.nan))]
+    quantiles = finite_sample.quantile([0.25, 0.5, 0.75]) if len(finite_sample) else pd.Series()
+    return {
+        "count": state.count,
+        "mean": _round_optional(state.mean if state.count else None),
+        "std": _round_optional(state.std),
+        "min": _round_optional(state.minimum),
+        "25%": _round_optional(quantiles.get(0.25)),
+        "50%": _round_optional(quantiles.get(0.5)),
+        "75%": _round_optional(quantiles.get(0.75)),
+        "max": _round_optional(state.maximum),
+    }
+
+
+def _sample_positions(rows: int, target_rows: int) -> np.ndarray:
+    count = min(rows, target_rows)
+    if count <= 0:
+        return np.empty(0, dtype=np.int64)
+    if count == rows:
+        return np.arange(rows, dtype=np.int64)
+    return np.unique(np.linspace(0, rows - 1, num=count, dtype=np.int64))
+
+
+def _sample_batch(batch, positions: np.ndarray, offset: int):
+    pa = _require_pyarrow()
+    if positions.size == 0 or batch.num_rows == 0:
+        return None
+    left = int(np.searchsorted(positions, offset, side="left"))
+    right = int(np.searchsorted(positions, offset + batch.num_rows, side="left"))
+    if right <= left:
+        return None
+    local = positions[left:right] - offset
+    return batch.take(pa.array(local, type=pa.int64())).to_pandas()
+
+
+def _duplicate_summary(sample: pd.DataFrame, rows: int) -> tuple[int, dict[str, Any]]:
+    if rows == len(sample):
+        count = int(sample.duplicated().sum())
+        return count, {
+            "method": "exact_streaming_sample",
+            "sampled": False,
+            "sample_rows": int(len(sample)),
+        }
+
+    sample_duplicates = int(sample.duplicated().sum())
+    rate = sample_duplicates / max(len(sample), 1)
+    estimate = int(round(rate * rows))
+    return estimate, {
+        "method": "streaming_sample_estimate",
+        "sampled": True,
+        "sample_rows": int(len(sample)),
+        "source_rows": int(rows),
+        "estimated_duplicate_rate": round(float(rate), 6),
+    }
+
+
+def _estimated_memory_usage(sample: pd.DataFrame, rows: int) -> tuple[float, dict[str, Any]]:
+    if len(sample) == 0:
+        return 0.0, {"method": "unavailable", "estimated": True}
+    sample_bytes = int(sample.memory_usage(index=True, deep=True).sum())
+    if len(sample) == rows:
+        estimated_bytes = sample_bytes
+        method = "exact_from_full_stream_sample"
+        estimated = False
+    else:
+        estimated_bytes = int(round(sample_bytes / len(sample) * rows))
+        method = "scaled_from_even_row_sample"
+        estimated = True
+    return round(estimated_bytes / (1024 * 1024), 3), {
+        "method": method,
+        "estimated": estimated,
+        "sample_rows": int(len(sample)),
+        "source_rows": int(rows),
+    }
+
+
+def build_streaming_profile(
+    source: StreamingDatasetSource,
+    *,
+    batch_size: int = STREAM_BATCH_SIZE,
+    sample_rows: int = STREAM_SAMPLE_ROWS,
+) -> dict[str, Any]:
+    """Profile a streaming Arrow source without materializing all rows."""
+    if batch_size < 1:
+        raise ValueError("batch_size must be at least 1.")
+    if sample_rows < 1:
+        raise ValueError("sample_rows must be at least 1.")
+
+    metadata = source.inspect()
+    rows = metadata.rows
+    if rows is None:
+        raise ValueError("Streaming profiling currently requires a source row count.")
+    if rows < 1:
+        raise ValueError(f"Dataset is empty: {metadata.name}")
+
+    schema_method = getattr(source, "schema", None)
+    if not callable(schema_method):
+        raise TypeError("Streaming profile sources must expose an Arrow schema().")
+    schema = schema_method()
+    columns = [field.name for field in schema]
+    numeric_cols, categorical_cols, date_cols = _column_groups(schema)
+    dtypes = {field.name: str(field.type) for field in schema}
+
+    positions = _sample_positions(int(rows), int(sample_rows))
+    sample_frames: list[pd.DataFrame] = []
+    preview_frame: pd.DataFrame | None = None
+    missing_counts = {column: 0 for column in columns}
+    selected_backend = resolve_numeric_backend()
+
+    native_accumulators: dict[str, Any] = {}
+    python_states = {column: NumericColumnState() for column in numeric_cols}
+    if selected_backend == "rust":
+        native_accumulators = {
+            column: create_numeric_accumulator(stream_id=index)
+            for index, column in enumerate(numeric_cols)
+        }
+
+    offset = 0
+    batches_scanned = 0
+    for batch in source.iter_batches(batch_size=batch_size):
+        batches_scanned += 1
+        if preview_frame is None:
+            preview_frame = batch.slice(0, min(15, batch.num_rows)).to_pandas()
+
+        sampled = _sample_batch(batch, positions, offset)
+        if sampled is not None and not sampled.empty:
+            sample_frames.append(sampled)
+
+        for index, column in enumerate(columns):
+            array = batch.column(index)
+            if column not in numeric_cols:
+                missing_counts[column] += int(array.null_count)
+                continue
+
+            values = _arrow_numeric_to_float64(array)
+            accumulator = native_accumulators.get(column)
+            if accumulator is not None:
+                accumulator.update_f64(values)
+            else:
+                payload = numeric_state(values, backend="numpy")
+                python_states[column] = python_states[column].merge(
+                    _state_from_payload(payload)
+                )
+
+        offset += int(batch.num_rows)
+
+    if offset != rows:
+        raise ValueError(
+            f"Streaming source metadata reported {rows} rows but yielded {offset}."
+        )
+
+    sample = (
+        pd.concat(sample_frames, ignore_index=True)
+        if sample_frames
+        else pd.DataFrame(columns=columns)
+    )
+    if len(sample) > len(positions):
+        sample = sample.head(len(positions))
+
+    numeric_summary: dict[str, dict[str, Any]] = {}
+    quantile_accuracy = None
+    if selected_backend == "rust":
+        for column in numeric_cols:
+            payload = dict(native_accumulators[column].snapshot())
+            numeric_summary[column] = _summary_from_native_snapshot(payload)
+            missing_counts[column] = int(payload["missing"])
+            quantile_accuracy = payload.get("quantiles", {}).get(
+                "relative_accuracy",
+                quantile_accuracy,
+            )
+        numeric_metadata = {
+            "backend": "rust",
+            "method": "native_streaming_accumulator",
+            "approximate_quantiles": True,
+            "quantile_relative_accuracy": quantile_accuracy,
+            "finite_only_moments": True,
+            "columns_profiled": len(numeric_cols),
+            "raw_observations_retained": False,
+        }
+    else:
+        for column in numeric_cols:
+            state = python_states[column]
+            numeric_summary[column] = _summary_from_python_state(
+                state,
+                sample[column] if column in sample else pd.Series(dtype="float64"),
+            )
+            missing_counts[column] = state.missing
+        numeric_metadata = {
+            "backend": "numpy",
+            "method": "mergeable_streaming_moments_with_row_sample_quantiles",
+            "approximate_quantiles": len(sample) < rows,
+            "quantile_sample_rows": int(len(sample)),
+            "finite_only_moments": True,
+            "columns_profiled": len(numeric_cols),
+            "raw_observations_retained": False,
+        }
+
+    missing_series = pd.Series(missing_counts, dtype="int64")
+    missing_percent = (missing_series / max(rows, 1) * 100).round(2)
+
+    categorical_summary = {}
+    for column in categorical_cols:
+        if column not in sample:
+            continue
+        counts = sample[column].value_counts(dropna=False).head(10)
+        categorical_summary[column] = {
+            "unique_values": int(sample[column].nunique(dropna=True)),
+            "top_values": {str(key): int(value) for key, value in counts.items()},
+        }
+    categorical_metadata = {
+        "method": "exact" if len(sample) == rows else "evenly_spaced_row_sample",
+        "sampled": len(sample) < rows,
+        "sample_rows": int(len(sample)),
+        "source_rows": int(rows),
+    }
+
+    duplicate_rows, duplicate_metadata = _duplicate_summary(sample, int(rows))
+    correlations, correlation_metadata = _bounded_correlations(sample, numeric_cols)
+    correlation_metadata.update({
+        "row_sampled": len(sample) < rows,
+        "sample_rows": int(len(sample)),
+        "source_rows": int(rows),
+    })
+
+    memory_usage_mb, memory_usage_metadata = _estimated_memory_usage(sample, int(rows))
+    preview_source = preview_frame if preview_frame is not None else pd.DataFrame(columns=columns)
+    preview_object = preview_source.astype(object)
+    preview = preview_object.where(preview_object.notna(), None).to_dict(orient="records")
+
+    source_metadata = metadata.to_dict() if hasattr(metadata, "to_dict") else asdict(metadata)
+    return {
+        "shape": {"rows": int(rows), "columns": len(columns)},
+        "columns": columns,
+        "dtypes": dtypes,
+        "numeric_columns": numeric_cols,
+        "categorical_columns": categorical_cols,
+        "date_columns": date_cols,
+        "missing_counts": series_to_dict(missing_series),
+        "missing_percent": series_to_dict(missing_percent),
+        "duplicate_rows": duplicate_rows,
+        "duplicate_percent": round(duplicate_rows / max(rows, 1) * 100, 2),
+        "duplicate_metadata": duplicate_metadata,
+        "memory_usage_mb": memory_usage_mb,
+        "memory_usage_metadata": memory_usage_metadata,
+        "numeric_summary": numeric_summary,
+        "numeric_summary_metadata": numeric_metadata,
+        "categorical_summary": categorical_summary,
+        "categorical_summary_metadata": categorical_metadata,
+        "correlations": correlations,
+        "correlation_metadata": correlation_metadata,
+        "preview": preview,
+        "source_metadata": source_metadata,
+        "streaming_metadata": {
+            "enabled": True,
+            "full_materialization": False,
+            "batch_size": int(batch_size),
+            "batches_scanned": int(batches_scanned),
+            "sample_rows": int(len(sample)),
+            "sample_strategy": "evenly_spaced_global_rows",
+            "numeric_backend": selected_backend,
+        },
+    }

From 6cf3a06c750a3a636b3c7e60e974dc41408708ea Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:27:43 +0530
Subject: [PATCH 162/439] feat: stream Parquet in focused profile API

---
 src/framevitals/focused.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index a42c9ce..7f1ee2d 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -13,7 +13,7 @@
 
 import pandas as pd
 
-from framevitals.sources import resolve_source
+from framevitals.sources import StreamingDatasetSource, resolve_source
 
 
 DataInput = str | Path | pd.DataFrame
@@ -37,10 +37,18 @@ def _named(payload: dict[str, Any], source_name: str) -> dict[str, Any]:
 
 
 def profile(data: DataInput) -> dict[str, Any]:
+    """Profile a dataset, streaming Arrow-capable file sources when available."""
+    source = resolve_source(data)
+    metadata = source.inspect()
+    if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
+        from framevitals.streaming_profile import build_streaming_profile
+
+        return _named(build_streaming_profile(source), metadata.name)
+
     from framevitals.profiler import build_profile
 
-    dataframe, source_name = _load(data)
-    return _named(build_profile(dataframe), source_name)
+    dataframe = source.load()
+    return _named(build_profile(dataframe), metadata.name)
 
 
 def roles(data: DataInput) -> dict[str, Any]:

From 32c05f464af0946fac2b43c911c2c0aef2316fda Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:28:08 +0530
Subject: [PATCH 163/439] test: cover bounded Parquet streaming profiles

---
 tests/test_parquet_streaming.py | 84 +++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 tests/test_parquet_streaming.py

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
new file mode 100644
index 0000000..c037b94
--- /dev/null
+++ b/tests/test_parquet_streaming.py
@@ -0,0 +1,84 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+pa = pytest.importorskip("pyarrow")
+pq = pytest.importorskip("pyarrow.parquet")
+
+import framevitals
+from framevitals.sources import ParquetSource, resolve_source
+
+
+def _write_parquet(path, rows: int = 10_000) -> pd.DataFrame:
+    frame = pd.DataFrame({
+        "value": np.arange(rows, dtype=np.float64),
+        "other": np.arange(rows, dtype=np.float64) * 2.0 + 3.0,
+        "group": [f"g-{index % 7}" for index in range(rows)],
+        "event_time": pd.date_range("2026-01-01", periods=rows, freq="min"),
+    })
+    frame.loc[::101, "value"] = np.nan
+    table = pa.Table.from_pandas(frame, preserve_index=False)
+    pq.write_table(table, path, row_group_size=777)
+    return frame
+
+
+def test_parquet_source_exposes_metadata_projection_and_batches(tmp_path):
+    path = tmp_path / "dataset.parquet"
+    frame = _write_parquet(path, rows=2_500)
+
+    source = resolve_source(path)
+    assert isinstance(source, ParquetSource)
+
+    metadata = source.inspect()
+    assert metadata.rows == len(frame)
+    assert metadata.columns == len(frame.columns)
+    assert metadata.format == "parquet"
+    assert metadata.materialized is False
+    assert metadata.supports_projection is True
+    assert metadata.supports_streaming is True
+
+    batches = list(source.iter_batches(batch_size=600, columns=["value", "group"]))
+    assert sum(batch.num_rows for batch in batches) == len(frame)
+    assert batches[0].schema.names == ["value", "group"]
+    assert max(batch.num_rows for batch in batches) <= 600
+
+
+def test_public_profile_streams_parquet_without_calling_load(tmp_path, monkeypatch):
+    path = tmp_path / "stream.parquet"
+    frame = _write_parquet(path, rows=12_000)
+
+    def fail_load(self):
+        raise AssertionError("streaming profile must not materialize the complete Parquet file")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.profile(path)
+
+    assert result["dataset_name"] == "stream.parquet"
+    assert result["shape"] == {"rows": len(frame), "columns": 4}
+    assert result["streaming_metadata"]["enabled"] is True
+    assert result["streaming_metadata"]["full_materialization"] is False
+    assert result["streaming_metadata"]["sample_rows"] == len(frame)
+    assert result["source_metadata"]["supports_streaming"] is True
+    assert result["missing_counts"]["value"] == int(frame["value"].isna().sum())
+    assert result["numeric_summary"]["value"]["count"] == int(frame["value"].notna().sum())
+    assert result["numeric_summary"]["other"]["mean"] == pytest.approx(
+        round(float(frame["other"].mean()), 3)
+    )
+    assert "group" in result["categorical_summary"]
+    assert "event_time" in result["date_columns"]
+
+
+def test_large_parquet_retains_only_bounded_row_sample(tmp_path):
+    path = tmp_path / "large.parquet"
+    frame = _write_parquet(path, rows=60_000)
+
+    result = framevitals.profile(path)
+
+    streaming = result["streaming_metadata"]
+    assert streaming["full_materialization"] is False
+    assert streaming["sample_rows"] == 50_000
+    assert streaming["sample_strategy"] == "evenly_spaced_global_rows"
+    assert result["categorical_summary_metadata"]["sampled"] is True
+    assert result["correlation_metadata"]["row_sampled"] is True
+    assert result["duplicate_metadata"]["sampled"] is True
+    assert result["missing_counts"]["value"] == int(frame["value"].isna().sum())

From 3e9382ca790440457467520e6c06614bb61f68ed Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:28:53 +0530
Subject: [PATCH 164/439] ci: validate Arrow streaming with and without native
 core

---
 .github/workflows/test.yml | 71 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 69 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1cefe34..8d0ff60 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -120,7 +120,11 @@ jobs:
       - name: Install native build tooling
         run: |
           python -m pip install --upgrade pip
-          pip install "maturin>=1.14,<2" "numpy>=1.26" "pandas>=2.2"
+          pip install \
+            "maturin>=1.14,<2" \
+            "numpy>=1.26" \
+            "pandas>=2.2" \
+            "pyarrow>=25.0,<26"
 
       - name: Build native Python wheel
         run: |
@@ -130,12 +134,19 @@ jobs:
             --interpreter python \
             --out dist-native
 
-      - name: Verify native Python bridge and profiler routing
+      - name: Verify native Python bridge and streaming profiler routing
         run: |
           pip install --force-reinstall --no-deps dist-native/*.whl
           FRAMEVITALS_BACKEND=rust python - <<'PY'
+          from pathlib import Path
+          from tempfile import TemporaryDirectory
+
           import numpy as np
           import pandas as pd
+          import pyarrow as pa
+          import pyarrow.parquet as pq
+
+          import framevitals
           from framevitals import _native
           from framevitals.profiler import build_profile
 
@@ -154,6 +165,15 @@ jobs:
           assert native_profile["quantiles"]["p50"] is not None
           assert native_profile["reservoir"]
 
+          accumulator = _native.NumericAccumulator(stream_id=29)
+          accumulator.update_f64(values[:3])
+          accumulator.update_f64(values[3:])
+          accumulated = accumulator.snapshot()
+          assert accumulator.observations == len(values)
+          assert accumulated["count"] == state["count"]
+          assert accumulated["missing"] == state["missing"]
+          assert accumulated["infinite"] == state["infinite"]
+
           public_profile = build_profile(pd.DataFrame({
               "value": [1.0, 2.0, None, 4.0, 4.0],
               "other": [10.0, 20.0, 30.0, 40.0, 50.0],
@@ -162,9 +182,56 @@ jobs:
           assert public_profile["numeric_summary_metadata"]["approximate_quantiles"] is True
           assert public_profile["missing_counts"]["value"] == 1
           assert public_profile["numeric_summary"]["value"]["count"] == 4
+
+          with TemporaryDirectory() as directory:
+              path = Path(directory) / "native-stream.parquet"
+              frame = pd.DataFrame({
+                  "value": np.arange(20_000, dtype=np.float64),
+                  "other": np.arange(20_000, dtype=np.float64) * 3.0,
+                  "group": [f"g-{index % 5}" for index in range(20_000)],
+              })
+              frame.loc[::97, "value"] = np.nan
+              pq.write_table(pa.Table.from_pandas(frame, preserve_index=False), path, row_group_size=777)
+              streamed = framevitals.profile(path)
+              assert streamed["streaming_metadata"]["full_materialization"] is False
+              assert streamed["streaming_metadata"]["numeric_backend"] == "rust"
+              assert streamed["numeric_summary_metadata"]["backend"] == "rust"
+              assert streamed["missing_counts"]["value"] == int(frame["value"].isna().sum())
+              assert streamed["numeric_summary"]["value"]["count"] == int(frame["value"].notna().sum())
+
           print(_native.backend_info())
           PY
 
+  arrow-streaming:
+    name: Arrow streaming (Python 3.12, NumPy fallback)
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install Arrow capability
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[arrow,dev]"
+
+      - name: Verify dependency consistency
+        run: python -m pip check
+
+      - name: Run streaming Parquet tests without native extension
+        env:
+          FRAMEVITALS_BACKEND: numpy
+        run: pytest -v tests/test_parquet_streaming.py
+
   optional-features:
     name: optional features (Python 3.12)
     runs-on: ubuntu-latest

From 144dd33aa41ce3aac43c4472280bb2a4aded6015 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:32:37 +0530
Subject: [PATCH 165/439] feat: reuse streaming row samples across planning and
 relationships

---
 src/framevitals/streaming_profile.py | 68 ++++++++++++++++++++++++++--
 1 file changed, 64 insertions(+), 4 deletions(-)

diff --git a/src/framevitals/streaming_profile.py b/src/framevitals/streaming_profile.py
index e9230cf..f676f03 100644
--- a/src/framevitals/streaming_profile.py
+++ b/src/framevitals/streaming_profile.py
@@ -7,6 +7,7 @@
 
 from __future__ import annotations
 
+from collections.abc import Sequence
 from dataclasses import asdict
 from typing import Any
 
@@ -70,6 +71,15 @@ def _column_groups(schema) -> tuple[list[str], list[str], list[str]]:
     return numeric, categorical, dates
 
 
+def numeric_columns_for_streaming_source(source: StreamingDatasetSource) -> list[str]:
+    """Return Arrow numeric column names without reading row data."""
+    schema_method = getattr(source, "schema", None)
+    if not callable(schema_method):
+        raise TypeError("Streaming profile sources must expose an Arrow schema().")
+    numeric, _, _ = _column_groups(schema_method())
+    return numeric
+
+
 def _arrow_numeric_to_float64(array) -> np.ndarray:
     """Convert one bounded Arrow numeric array to a contiguous float64 buffer."""
     try:
@@ -131,7 +141,8 @@ def _summary_from_python_state(
     sample: pd.Series,
 ) -> dict[str, Any]:
     finite_sample = pd.to_numeric(sample, errors="coerce")
-    finite_sample = finite_sample[np.isfinite(finite_sample.to_numpy(dtype="float64", na_value=np.nan))]
+    finite_values = finite_sample.to_numpy(dtype="float64", na_value=np.nan)
+    finite_sample = finite_sample[np.isfinite(finite_values)]
     quantiles = finite_sample.quantile([0.25, 0.5, 0.75]) if len(finite_sample) else pd.Series()
     return {
         "count": state.count,
@@ -166,6 +177,46 @@ def _sample_batch(batch, positions: np.ndarray, offset: int):
     return batch.take(pa.array(local, type=pa.int64())).to_pandas()
 
 
+def sample_streaming_source(
+    source: StreamingDatasetSource,
+    *,
+    sample_rows: int,
+    batch_size: int = STREAM_BATCH_SIZE,
+    columns: Sequence[str] | None = None,
+) -> pd.DataFrame:
+    """Return an evenly spaced bounded row sample without full materialization."""
+    if sample_rows < 1:
+        raise ValueError("sample_rows must be at least 1.")
+    if batch_size < 1:
+        raise ValueError("batch_size must be at least 1.")
+
+    metadata = source.inspect()
+    rows = metadata.rows
+    if rows is None:
+        raise ValueError("Streaming sampling requires a source row count.")
+    if rows < 1:
+        raise ValueError(f"Dataset is empty: {metadata.name}")
+
+    positions = _sample_positions(int(rows), int(sample_rows))
+    frames: list[pd.DataFrame] = []
+    offset = 0
+    for batch in source.iter_batches(batch_size=batch_size, columns=columns):
+        sampled = _sample_batch(batch, positions, offset)
+        if sampled is not None and not sampled.empty:
+            frames.append(sampled)
+        offset += int(batch.num_rows)
+
+    if offset != rows:
+        raise ValueError(
+            f"Streaming source metadata reported {rows} rows but yielded {offset}."
+        )
+
+    if frames:
+        sample = pd.concat(frames, ignore_index=True)
+        return sample.head(len(positions))
+    return pd.DataFrame(columns=list(columns or ()))
+
+
 def _duplicate_summary(sample: pd.DataFrame, rows: int) -> tuple[int, dict[str, Any]]:
     if rows == len(sample):
         count = int(sample.duplicated().sum())
@@ -212,8 +263,14 @@ def build_streaming_profile(
     *,
     batch_size: int = STREAM_BATCH_SIZE,
     sample_rows: int = STREAM_SAMPLE_ROWS,
-) -> dict[str, Any]:
-    """Profile a streaming Arrow source without materializing all rows."""
+    return_sample: bool = False,
+) -> dict[str, Any] | tuple[dict[str, Any], pd.DataFrame]:
+    """Profile a streaming Arrow source without materializing all rows.
+
+    ``return_sample=True`` lets planning/orchestration reuse the bounded row
+    sample generated during the same full-file scan instead of reading the
+    source a second time.
+    """
     if batch_size < 1:
         raise ValueError("batch_size must be at least 1.")
     if sample_rows < 1:
@@ -361,7 +418,7 @@ def build_streaming_profile(
     preview = preview_object.where(preview_object.notna(), None).to_dict(orient="records")
 
     source_metadata = metadata.to_dict() if hasattr(metadata, "to_dict") else asdict(metadata)
-    return {
+    payload = {
         "shape": {"rows": int(rows), "columns": len(columns)},
         "columns": columns,
         "dtypes": dtypes,
@@ -393,3 +450,6 @@ def build_streaming_profile(
             "numeric_backend": selected_backend,
         },
     }
+    if return_sample:
+        return payload, sample
+    return payload

From e677694101f94f2719d300bfaea6540d55fb4490 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:32:58 +0530
Subject: [PATCH 166/439] feat: let dataset signals use full source shape with
 sampled rows

---
 src/framevitals/dataset_signals.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/framevitals/dataset_signals.py b/src/framevitals/dataset_signals.py
index 660600e..8f25a1d 100644
--- a/src/framevitals/dataset_signals.py
+++ b/src/framevitals/dataset_signals.py
@@ -45,14 +45,24 @@ def detect_dataset_signals(
     df,
     profile: dict,
     column_roles: dict | None = None,
+    source_shape: tuple[int, int] | None = None,
 ) -> dict:
     """Produce structural signals, reusing cached pipeline metadata when supplied.
 
     ``column_roles`` is optional for backward compatibility. The main analysis
     pipeline passes its already-computed role map so this stage does not repeat
     per-column semantic scans.
+
+    ``source_shape`` lets streaming/planning callers infer value-based roles on
+    a bounded row sample while preserving true full-dataset row/column counts
+    for scale signals and missingness percentages.
     """
-    rows, cols = df.shape
+    if source_shape is None:
+        rows, cols = df.shape
+    else:
+        rows, cols = (int(source_shape[0]), int(source_shape[1]))
+        if rows < 0 or cols < 0:
+            raise ValueError("source_shape values must be non-negative.")
 
     numeric_cols = profile.get("numeric_columns", [])
     categorical_cols = profile.get("categorical_columns", [])
@@ -87,7 +97,6 @@ def detect_dataset_signals(
     target_candidates = get_columns_with_role(column_roles, "target_candidate")
     long_text = get_columns_with_role(column_roles, "long_text")
 
-    # Semantic/value-pattern roles are already computed during role inference.
     email_like = get_columns_with_role(column_roles, "email_like")
     url_like = get_columns_with_role(column_roles, "url_like")
     uuid_like = get_columns_with_role(column_roles, "uuid_like")
@@ -103,7 +112,8 @@ def detect_dataset_signals(
         column_roles,
     )
 
-    lower_map = {str(c).lower(): c for c in df.columns}
+    source_columns = profile.get("columns", list(df.columns))
+    lower_map = {str(column).lower(): column for column in source_columns}
     has_bid_ask = "bid" in lower_map and "ask" in lower_map
     has_time_series = len(time_like) > 0 and len(numeric_cols) >= 1 and rows >= 20
 

From 7ce101ce7a5899d3ce15373d8abfff90f2398a3e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:33:35 +0530
Subject: [PATCH 167/439] feat: make planning source-aware for streaming
 datasets

---
 src/framevitals/planning_api.py | 96 ++++++++++++++++++++++++++++++---
 1 file changed, 90 insertions(+), 6 deletions(-)

diff --git a/src/framevitals/planning_api.py b/src/framevitals/planning_api.py
index da69621..1230c7f 100644
--- a/src/framevitals/planning_api.py
+++ b/src/framevitals/planning_api.py
@@ -20,10 +20,61 @@
 from framevitals.execution import derive_execution_budget
 from framevitals.planning import AnalysisPlan
 from framevitals.profiler import build_profile
-from framevitals.sources import resolve_source
+from framevitals.sources import StreamingDatasetSource, resolve_source
 
 
 DataInput = str | Path | pd.DataFrame
+PLANNING_SAMPLE_ROWS = 5_000
+
+
+def _streaming_head_sample(
+    source: StreamingDatasetSource,
+    *,
+    max_rows: int = PLANNING_SAMPLE_ROWS,
+) -> pd.DataFrame:
+    """Read at most ``max_rows`` rows and stop; planning must not scan the file."""
+    frames: list[pd.DataFrame] = []
+    remaining = int(max_rows)
+    for batch in source.iter_batches(batch_size=max_rows):
+        if remaining <= 0:
+            break
+        take = min(remaining, int(batch.num_rows))
+        if take > 0:
+            frames.append(batch.slice(0, take).to_pandas())
+            remaining -= take
+        if remaining <= 0:
+            break
+    if not frames:
+        raise ValueError("Streaming dataset produced no rows for planning.")
+    return pd.concat(frames, ignore_index=True)
+
+
+def _project_sample_profile_to_source(
+    profile: dict[str, Any],
+    *,
+    source_rows: int,
+    source_columns: int,
+    sample_rows: int,
+) -> dict[str, Any]:
+    """Scale rate-based sample metrics to the true source shape for planning."""
+    projected = dict(profile)
+    factor = source_rows / max(sample_rows, 1)
+    projected["shape"] = {"rows": int(source_rows), "columns": int(source_columns)}
+    projected["missing_counts"] = {
+        column: int(round(float(value or 0) * factor))
+        for column, value in profile.get("missing_counts", {}).items()
+    }
+    sample_duplicate_rows = int(profile.get("duplicate_rows", 0) or 0)
+    projected["duplicate_rows"] = int(round(sample_duplicate_rows * factor))
+    projected["planning_sample_metadata"] = {
+        "sampled": sample_rows < source_rows,
+        "sample_rows": int(sample_rows),
+        "source_rows": int(source_rows),
+        "strategy": "bounded_head",
+        "full_scan": False,
+        "rate_metrics_projected": sample_rows < source_rows,
+    }
+    return projected
 
 
 def plan(
@@ -49,22 +100,54 @@ def plan(
 
     source = resolve_source(data)
     source_metadata = source.inspect()
-    dataframe = source.load()
 
-    if resolved.target is not None and resolved.target not in dataframe.columns:
+    if source_metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
+        dataframe = _streaming_head_sample(source)
+        source_rows = int(source_metadata.rows or len(dataframe))
+        source_columns = int(source_metadata.columns or len(dataframe.columns))
+        dataset_profile = _project_sample_profile_to_source(
+            build_profile(dataframe),
+            source_rows=source_rows,
+            source_columns=source_columns,
+            sample_rows=len(dataframe),
+        )
+        planning_data = {
+            "materialized_full_dataset": False,
+            "sampled": len(dataframe) < source_rows,
+            "sample_rows": int(len(dataframe)),
+            "source_rows": source_rows,
+            "strategy": "bounded_head",
+            "full_scan": False,
+        }
+    else:
+        dataframe = source.load()
+        source_rows = int(len(dataframe))
+        source_columns = int(len(dataframe.columns))
+        dataset_profile = build_profile(dataframe)
+        planning_data = {
+            "materialized_full_dataset": True,
+            "sampled": False,
+            "sample_rows": source_rows,
+            "source_rows": source_rows,
+            "strategy": "full_dataset",
+            "full_scan": True,
+        }
+
+    source_columns_list = list(dataset_profile.get("columns", dataframe.columns))
+    if resolved.target is not None and resolved.target not in source_columns_list:
         raise ValueError(f"Target column not found: {resolved.target}")
 
-    dataset_profile = build_profile(dataframe)
     column_roles = infer_column_roles(dataframe)
     dataset_signals = detect_dataset_signals(
         dataframe,
         dataset_profile,
         column_roles=column_roles,
+        source_shape=(source_rows, source_columns),
     )
 
     budget = derive_execution_budget(
-        len(dataframe),
-        len(dataframe.columns),
+        source_rows,
+        source_columns,
         mode=resolved.mode,
     )
 
@@ -89,6 +172,7 @@ def plan(
     return AnalysisPlan({
         "dataset_name": source_metadata.name,
         "source": source_metadata.to_dict(),
+        "planning_data": planning_data,
         "analysis_mode": resolved.mode,
         "target": resolved.target,
         "shape": dict(dataset_profile.get("shape", {})),

From cba668ac44084796b1c0f2019c86c6462913ac10 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:34:02 +0530
Subject: [PATCH 168/439] feat: sample streaming sources for sparse
 relationships

---
 src/framevitals/focused.py | 38 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index 7f1ee2d..633112b 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -174,7 +174,41 @@ def relationships(
 ) -> dict[str, Any]:
     from framevitals.relationship_graph import build_numeric_relationship_graph
 
-    dataframe, source_name = _load(data)
+    source = resolve_source(data)
+    metadata = source.inspect()
+    if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
+        from framevitals.streaming_profile import (
+            numeric_columns_for_streaming_source,
+            sample_streaming_source,
+        )
+
+        numeric_columns = numeric_columns_for_streaming_source(source)
+        dataframe = sample_streaming_source(
+            source,
+            sample_rows=max_sample_rows,
+            columns=numeric_columns,
+        )
+        payload = build_numeric_relationship_graph(
+            dataframe,
+            max_sample_rows=max_sample_rows,
+            projections=projections,
+            min_abs_correlation=min_abs_correlation,
+            max_candidate_pairs=max_candidate_pairs,
+            max_edges_returned=max_edges_returned,
+        )
+        if payload.get("available"):
+            sample = payload.setdefault("sample", {})
+            sample.update({
+                "source_rows": int(metadata.rows or len(dataframe)),
+                "sample_rows": int(len(dataframe)),
+                "sampled": int(metadata.rows or len(dataframe)) > len(dataframe),
+                "strategy": "streaming_evenly_spaced_global_rows",
+                "full_materialization": False,
+            })
+            payload["source"] = metadata.to_dict()
+        return _named(payload, metadata.name)
+
+    dataframe = source.load()
     payload = build_numeric_relationship_graph(
         dataframe,
         max_sample_rows=max_sample_rows,
@@ -183,7 +217,7 @@ def relationships(
         max_candidate_pairs=max_candidate_pairs,
         max_edges_returned=max_edges_returned,
     )
-    return _named(payload, source_name)
+    return _named(payload, metadata.name)
 
 
 def target_analysis(

From ea4f2451b1476e8320864ece8a14e38313c20eb0 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:34:28 +0530
Subject: [PATCH 169/439] test: cover source-aware Parquet planning and
 relationships

---
 tests/test_parquet_streaming.py | 46 +++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
index c037b94..4b79e96 100644
--- a/tests/test_parquet_streaming.py
+++ b/tests/test_parquet_streaming.py
@@ -82,3 +82,49 @@ def test_large_parquet_retains_only_bounded_row_sample(tmp_path):
     assert result["correlation_metadata"]["row_sampled"] is True
     assert result["duplicate_metadata"]["sampled"] is True
     assert result["missing_counts"]["value"] == int(frame["value"].isna().sum())
+
+
+def test_plan_reads_only_bounded_parquet_sample_and_uses_true_shape(tmp_path, monkeypatch):
+    path = tmp_path / "plan.parquet"
+    frame = _write_parquet(path, rows=12_000)
+
+    def fail_load(self):
+        raise AssertionError("plan must not materialize the complete Parquet file")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.plan(path, mode="standard")
+
+    assert result["shape"] == {"rows": len(frame), "columns": 4}
+    assert result["signals"]["row_count"] == len(frame)
+    assert result["signals"]["column_count"] == 4
+    assert result["execution_budget"]["rows"] == len(frame)
+    planning = result["planning_data"]
+    assert planning["materialized_full_dataset"] is False
+    assert planning["full_scan"] is False
+    assert planning["sampled"] is True
+    assert planning["sample_rows"] == 5_000
+    assert result["source"]["supports_streaming"] is True
+
+
+def test_relationships_stream_numeric_projection_without_full_materialization(
+    tmp_path,
+    monkeypatch,
+):
+    path = tmp_path / "relationships.parquet"
+    frame = _write_parquet(path, rows=12_000)
+
+    def fail_load(self):
+        raise AssertionError("relationships must not materialize the complete Parquet file")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.relationships(path, max_sample_rows=256)
+
+    assert result["available"] is True
+    assert result["sample"]["source_rows"] == len(frame)
+    assert result["sample"]["sample_rows"] <= 256
+    assert result["sample"]["sampled"] is True
+    assert result["sample"]["full_materialization"] is False
+    assert result["sample"]["strategy"] == "streaming_evenly_spaced_global_rows"
+    assert result["source"]["supports_projection"] is True
+    edge_pairs = {(edge["source"], edge["target"]) for edge in result["edges"]}
+    assert ("value", "other") in edge_pairs or ("other", "value") in edge_pairs

From 23daf89f0467034c30f2e1b25f6df57beb287ddc Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:35:08 +0530
Subject: [PATCH 170/439] bench: add reproducible 100k by 105 profiling harness

---
 benchmarks/benchmark_profile_scale.py | 305 ++++++++++++++++++++++++++
 1 file changed, 305 insertions(+)
 create mode 100644 benchmarks/benchmark_profile_scale.py

diff --git a/benchmarks/benchmark_profile_scale.py b/benchmarks/benchmark_profile_scale.py
new file mode 100644
index 0000000..945bee7
--- /dev/null
+++ b/benchmarks/benchmark_profile_scale.py
@@ -0,0 +1,305 @@
+#!/usr/bin/env python3
+"""Reproducible FrameVitals scale benchmark.
+
+The harness runs each scenario in a fresh subprocess so process peak RSS values
+are comparable and one scenario's allocator state cannot contaminate another.
+It intentionally records measurements rather than enforcing fragile absolute
+performance thresholds in CI.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import platform
+import resource
+import subprocess
+import sys
+import tempfile
+import time
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+
+DEFAULT_ROWS = 100_000
+DEFAULT_NUMERIC_COLUMNS = 100
+DEFAULT_CATEGORICAL_COLUMNS = 5
+DEFAULT_SEED = 42
+SCENARIOS = ("numpy", "auto", "parquet")
+
+
+def _peak_rss_mb() -> float:
+    raw = float(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
+    # Linux reports KiB; macOS reports bytes.
+    if platform.system() == "Darwin":
+        return raw / (1024 * 1024)
+    return raw / 1024
+
+
+def _frame(
+    rows: int,
+    numeric_columns: int,
+    categorical_columns: int,
+    seed: int,
+) -> pd.DataFrame:
+    rng = np.random.default_rng(seed)
+    values = rng.standard_normal((rows, numeric_columns), dtype=np.float64)
+    if numeric_columns:
+        values[::97, ::7] = np.nan
+    frame = pd.DataFrame(
+        values,
+        columns=[f"metric_{index:03d}" for index in range(numeric_columns)],
+        copy=False,
+    )
+    for index in range(categorical_columns):
+        cardinality = 7 + index * 3
+        frame[f"category_{index:02d}"] = np.asarray(
+            [f"g{index}_{row % cardinality}" for row in range(rows)],
+            dtype=object,
+        )
+    return frame
+
+
+def _write_parquet_streaming(
+    path: Path,
+    *,
+    rows: int,
+    numeric_columns: int,
+    categorical_columns: int,
+    seed: int,
+    chunk_rows: int = 20_000,
+) -> None:
+    try:
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+    except ImportError as exc:
+        raise RuntimeError(
+            'Parquet benchmark requires: pip install "framevitals[arrow]"'
+        ) from exc
+
+    writer = None
+    try:
+        offset = 0
+        chunk_index = 0
+        while offset < rows:
+            current = min(chunk_rows, rows - offset)
+            frame = _frame(
+                current,
+                numeric_columns,
+                categorical_columns,
+                seed + chunk_index,
+            )
+            table = pa.Table.from_pandas(frame, preserve_index=False)
+            if writer is None:
+                writer = pq.ParquetWriter(path, table.schema)
+            writer.write_table(table, row_group_size=min(current, 10_000))
+            offset += current
+            chunk_index += 1
+    finally:
+        if writer is not None:
+            writer.close()
+
+
+def _result_metadata(result: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "shape": result.get("shape"),
+        "numeric_summary_metadata": result.get("numeric_summary_metadata"),
+        "correlation_metadata": result.get("correlation_metadata"),
+        "duplicate_metadata": result.get("duplicate_metadata"),
+        "streaming_metadata": result.get("streaming_metadata"),
+        "source_metadata": result.get("source_metadata"),
+    }
+
+
+def _worker(args: argparse.Namespace) -> int:
+    import framevitals
+    from framevitals.backends import backend_status
+
+    started = time.perf_counter()
+    source_memory_mb = None
+
+    if args.scenario in {"numpy", "auto"}:
+        if args.scenario == "numpy":
+            os.environ["FRAMEVITALS_BACKEND"] = "numpy"
+        else:
+            os.environ.pop("FRAMEVITALS_BACKEND", None)
+        frame = _frame(
+            args.rows,
+            args.numeric_columns,
+            args.categorical_columns,
+            args.seed,
+        )
+        source_memory_mb = round(
+            float(frame.memory_usage(index=True, deep=True).sum()) / (1024 * 1024),
+            3,
+        )
+        result = framevitals.profile(frame)
+    elif args.scenario == "parquet":
+        if not args.parquet_path:
+            raise ValueError("parquet worker requires --parquet-path")
+        os.environ.pop("FRAMEVITALS_BACKEND", None)
+        result = framevitals.profile(Path(args.parquet_path))
+    else:
+        raise ValueError(f"Unknown benchmark scenario: {args.scenario}")
+
+    elapsed = time.perf_counter() - started
+    payload = {
+        "scenario": args.scenario,
+        "rows": args.rows,
+        "numeric_columns": args.numeric_columns,
+        "categorical_columns": args.categorical_columns,
+        "total_columns": args.numeric_columns + args.categorical_columns,
+        "seed": args.seed,
+        "elapsed_seconds": round(elapsed, 6),
+        "peak_rss_mb": round(_peak_rss_mb(), 3),
+        "source_memory_mb": source_memory_mb,
+        "backend_status": backend_status(),
+        "result": _result_metadata(result),
+        "python": platform.python_version(),
+        "platform": platform.platform(),
+    }
+    print(json.dumps(payload, sort_keys=True))
+    return 0
+
+
+def _run_scenario(
+    script: Path,
+    scenario: str,
+    *,
+    rows: int,
+    numeric_columns: int,
+    categorical_columns: int,
+    seed: int,
+    parquet_path: Path | None,
+) -> dict[str, Any]:
+    command = [
+        sys.executable,
+        str(script),
+        "--worker",
+        "--scenario",
+        scenario,
+        "--rows",
+        str(rows),
+        "--numeric-columns",
+        str(numeric_columns),
+        "--categorical-columns",
+        str(categorical_columns),
+        "--seed",
+        str(seed),
+    ]
+    if parquet_path is not None:
+        command.extend(["--parquet-path", str(parquet_path)])
+
+    completed = subprocess.run(
+        command,
+        check=True,
+        capture_output=True,
+        text=True,
+        env=os.environ.copy(),
+    )
+    lines = [line for line in completed.stdout.splitlines() if line.strip()]
+    if not lines:
+        raise RuntimeError(f"Benchmark worker {scenario} produced no JSON output.")
+    return json.loads(lines[-1])
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--rows", type=int, default=DEFAULT_ROWS)
+    parser.add_argument("--numeric-columns", type=int, default=DEFAULT_NUMERIC_COLUMNS)
+    parser.add_argument(
+        "--categorical-columns",
+        type=int,
+        default=DEFAULT_CATEGORICAL_COLUMNS,
+    )
+    parser.add_argument("--seed", type=int, default=DEFAULT_SEED)
+    parser.add_argument(
+        "--scenarios",
+        nargs="+",
+        choices=SCENARIOS,
+        default=list(SCENARIOS),
+    )
+    parser.add_argument("--output", type=Path, default=None)
+    parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS)
+    parser.add_argument("--scenario", choices=SCENARIOS, default=None, help=argparse.SUPPRESS)
+    parser.add_argument("--parquet-path", default=None, help=argparse.SUPPRESS)
+    args = parser.parse_args()
+    if args.rows < 1:
+        parser.error("--rows must be at least 1")
+    if args.numeric_columns < 0 or args.categorical_columns < 0:
+        parser.error("column counts must be non-negative")
+    if args.numeric_columns + args.categorical_columns < 1:
+        parser.error("at least one column is required")
+    if args.worker and args.scenario is None:
+        parser.error("--worker requires --scenario")
+    return args
+
+
+def main() -> int:
+    args = _parse_args()
+    if args.worker:
+        return _worker(args)
+
+    script = Path(__file__).resolve()
+    results: list[dict[str, Any]] = []
+    with tempfile.TemporaryDirectory(prefix="framevitals-benchmark-") as directory:
+        parquet_path: Path | None = None
+        if "parquet" in args.scenarios:
+            parquet_path = Path(directory) / "scale.parquet"
+            prepare_started = time.perf_counter()
+            _write_parquet_streaming(
+                parquet_path,
+                rows=args.rows,
+                numeric_columns=args.numeric_columns,
+                categorical_columns=args.categorical_columns,
+                seed=args.seed,
+            )
+            parquet_prepare_seconds = round(time.perf_counter() - prepare_started, 6)
+        else:
+            parquet_prepare_seconds = None
+
+        for scenario in args.scenarios:
+            results.append(
+                _run_scenario(
+                    script,
+                    scenario,
+                    rows=args.rows,
+                    numeric_columns=args.numeric_columns,
+                    categorical_columns=args.categorical_columns,
+                    seed=args.seed,
+                    parquet_path=parquet_path if scenario == "parquet" else None,
+                )
+            )
+
+    payload = {
+        "benchmark_schema_version": 1,
+        "workload": {
+            "rows": args.rows,
+            "numeric_columns": args.numeric_columns,
+            "categorical_columns": args.categorical_columns,
+            "total_columns": args.numeric_columns + args.categorical_columns,
+            "seed": args.seed,
+        },
+        "parquet_prepare_seconds": parquet_prepare_seconds,
+        "measurements": results,
+        "notes": [
+            "Each measurement runs in a fresh process.",
+            "Peak RSS includes imports and in-memory source construction for DataFrame scenarios.",
+            "Parquet file preparation occurs outside the measured Parquet worker.",
+            "No absolute timing threshold is enforced; compare like-for-like machines and commits.",
+        ],
+    }
+    serialized = json.dumps(payload, indent=2, sort_keys=True)
+    if args.output is not None:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(serialized + "\n", encoding="utf-8")
+    print(serialized)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

From afe5e7ef1ee81d9e6c1102e6dbe3cddb3220a9c3 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:36:44 +0530
Subject: [PATCH 171/439] test: smoke test scale benchmark harness

---
 tests/test_benchmark_harness.py | 42 +++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 tests/test_benchmark_harness.py

diff --git a/tests/test_benchmark_harness.py b/tests/test_benchmark_harness.py
new file mode 100644
index 0000000..0a5ed97
--- /dev/null
+++ b/tests/test_benchmark_harness.py
@@ -0,0 +1,42 @@
+import json
+from pathlib import Path
+import subprocess
+import sys
+
+
+def test_scale_benchmark_harness_emits_machine_readable_result(tmp_path):
+    output = tmp_path / "benchmark.json"
+    script = Path("benchmarks/benchmark_profile_scale.py")
+
+    completed = subprocess.run(
+        [
+            sys.executable,
+            str(script),
+            "--rows",
+            "200",
+            "--numeric-columns",
+            "4",
+            "--categorical-columns",
+            "1",
+            "--scenarios",
+            "numpy",
+            "--output",
+            str(output),
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    payload = json.loads(output.read_text(encoding="utf-8"))
+    assert payload["benchmark_schema_version"] == 1
+    assert payload["workload"]["rows"] == 200
+    assert payload["workload"]["total_columns"] == 5
+    assert len(payload["measurements"]) == 1
+
+    measurement = payload["measurements"][0]
+    assert measurement["scenario"] == "numpy"
+    assert measurement["elapsed_seconds"] >= 0
+    assert measurement["peak_rss_mb"] > 0
+    assert measurement["result"]["shape"] == {"rows": 200, "columns": 5}
+    assert json.loads(completed.stdout)["workload"] == payload["workload"]

From 20f1d658df94d6518416a55ecf4d663db7e20dc8 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:38:02 +0530
Subject: [PATCH 172/439] feat(native): add bounded categorical string sketches

---
 .../src/categorical_sketches.rs               | 289 ++++++++++++++++++
 1 file changed, 289 insertions(+)
 create mode 100644 rust/framevitals-core/src/categorical_sketches.rs

diff --git a/rust/framevitals-core/src/categorical_sketches.rs b/rust/framevitals-core/src/categorical_sketches.rs
new file mode 100644
index 0000000..cb34755
--- /dev/null
+++ b/rust/framevitals-core/src/categorical_sketches.rs
@@ -0,0 +1,289 @@
+//! Mergeable bounded-memory sketches for UTF-8 categorical values.
+
+use std::collections::BTreeMap;
+
+use crate::sketches::{mix64, HyperLogLog};
+
+const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325;
+const FNV_PRIME: u64 = 0x0000_0100_0000_01b3;
+const DEFAULT_HEAVY_HITTER_CAPACITY: usize = 32;
+const DEFAULT_LABEL_BYTES: usize = 256;
+
+/// Incremental stable byte hasher used by Arrow buffer consumers.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct StableByteHasher {
+    hash: u64,
+    length: u64,
+}
+
+impl StableByteHasher {
+    #[must_use]
+    pub fn new() -> Self {
+        Self {
+            hash: FNV_OFFSET_BASIS,
+            length: 0,
+        }
+    }
+
+    pub fn update(&mut self, byte: u8) {
+        self.hash ^= u64::from(byte);
+        self.hash = self.hash.wrapping_mul(FNV_PRIME);
+        self.length = self.length.wrapping_add(1);
+    }
+
+    #[must_use]
+    pub fn finish(self) -> u64 {
+        mix64(self.hash ^ mix64(self.length))
+    }
+}
+
+impl Default for StableByteHasher {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct StringCandidate {
+    label: Vec<u8>,
+    count: u64,
+}
+
+/// Misra-Gries style bounded candidate tracker keyed by a stable full-value hash.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct StringHeavyHittersSketch {
+    capacity: usize,
+    max_label_bytes: usize,
+    counters: BTreeMap<u64, StringCandidate>,
+}
+
+impl StringHeavyHittersSketch {
+    #[must_use]
+    pub fn new(capacity: usize, max_label_bytes: usize) -> Self {
+        assert!(capacity > 0, "heavy-hitter capacity must be positive");
+        assert!(max_label_bytes > 0, "max_label_bytes must be positive");
+        Self {
+            capacity,
+            max_label_bytes,
+            counters: BTreeMap::new(),
+        }
+    }
+
+    pub fn observe_hashed(&mut self, hash: u64, label: &[u8], weight: u64) {
+        if weight == 0 {
+            return;
+        }
+        if let Some(candidate) = self.counters.get_mut(&hash) {
+            candidate.count = candidate.count.saturating_add(weight);
+            return;
+        }
+        if self.counters.len() < self.capacity {
+            self.counters.insert(
+                hash,
+                StringCandidate {
+                    label: label[..label.len().min(self.max_label_bytes)].to_vec(),
+                    count: weight,
+                },
+            );
+            return;
+        }
+
+        let minimum = self
+            .counters
+            .values()
+            .map(|candidate| candidate.count)
+            .min()
+            .unwrap_or(0);
+        let decrement = minimum.min(weight);
+        self.counters.retain(|_, candidate| {
+            candidate.count -= decrement;
+            candidate.count > 0
+        });
+
+        let remaining = weight - decrement;
+        if remaining > 0 {
+            self.counters.insert(
+                hash,
+                StringCandidate {
+                    label: label[..label.len().min(self.max_label_bytes)].to_vec(),
+                    count: remaining,
+                },
+            );
+        }
+    }
+
+    #[must_use]
+    pub fn merge(mut self, other: Self) -> Self {
+        assert_eq!(self.capacity, other.capacity, "heavy-hitter capacity mismatch");
+        assert_eq!(
+            self.max_label_bytes, other.max_label_bytes,
+            "heavy-hitter label limit mismatch"
+        );
+        for (hash, candidate) in other.counters {
+            self.observe_hashed(hash, &candidate.label, candidate.count);
+        }
+        self
+    }
+
+    #[must_use]
+    pub fn candidates(&self) -> Vec<(String, u64)> {
+        let mut values: Vec<(String, u64)> = self
+            .counters
+            .values()
+            .map(|candidate| {
+                (
+                    String::from_utf8_lossy(&candidate.label).into_owned(),
+                    candidate.count,
+                )
+            })
+            .collect();
+        values.sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0)));
+        values
+    }
+
+    #[must_use]
+    pub fn capacity(&self) -> usize {
+        self.capacity
+    }
+
+    #[must_use]
+    pub fn max_label_bytes(&self) -> usize {
+        self.max_label_bytes
+    }
+}
+
+impl Default for StringHeavyHittersSketch {
+    fn default() -> Self {
+        Self::new(DEFAULT_HEAVY_HITTER_CAPACITY, DEFAULT_LABEL_BYTES)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct CategoricalSketchState {
+    pub count: u64,
+    pub missing: u64,
+    pub cardinality: HyperLogLog,
+    pub heavy_hitters: StringHeavyHittersSketch,
+}
+
+impl Default for CategoricalSketchState {
+    fn default() -> Self {
+        Self {
+            count: 0,
+            missing: 0,
+            cardinality: HyperLogLog::default(),
+            heavy_hitters: StringHeavyHittersSketch::default(),
+        }
+    }
+}
+
+impl CategoricalSketchState {
+    pub fn observe_missing(&mut self) {
+        self.missing = self.missing.saturating_add(1);
+    }
+
+    pub fn observe_hashed(&mut self, hash: u64, label: &[u8]) {
+        self.count = self.count.saturating_add(1);
+        self.cardinality.observe_hash(hash);
+        self.heavy_hitters.observe_hashed(hash, label, 1);
+    }
+
+    pub fn observe_bytes(&mut self, value: Option<&[u8]>) {
+        let Some(value) = value else {
+            self.observe_missing();
+            return;
+        };
+        let mut hasher = StableByteHasher::new();
+        for &byte in value {
+            hasher.update(byte);
+        }
+        self.observe_hashed(hasher.finish(), value);
+    }
+
+    #[must_use]
+    pub fn merge(self, other: Self) -> Self {
+        Self {
+            count: self.count.saturating_add(other.count),
+            missing: self.missing.saturating_add(other.missing),
+            cardinality: self.cardinality.merge(other.cardinality),
+            heavy_hitters: self.heavy_hitters.merge(other.heavy_hitters),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{CategoricalSketchState, StableByteHasher, StringHeavyHittersSketch};
+
+    #[test]
+    fn stable_hash_is_incremental_and_deterministic() {
+        let mut left = StableByteHasher::new();
+        for byte in b"framevitals" {
+            left.update(*byte);
+        }
+        let mut right = StableByteHasher::new();
+        for chunk in [b"frame".as_slice(), b"vitals".as_slice()] {
+            for byte in chunk {
+                right.update(*byte);
+            }
+        }
+        assert_eq!(left.finish(), right.finish());
+    }
+
+    #[test]
+    fn categorical_state_tracks_cardinality_missing_and_heavy_hitters() {
+        let mut state = CategoricalSketchState::default();
+        for _ in 0..100 {
+            state.observe_bytes(Some(b"pune"));
+        }
+        for _ in 0..40 {
+            state.observe_bytes(Some(b"mumbai"));
+        }
+        state.observe_bytes(Some(b"nagpur"));
+        state.observe_bytes(None);
+
+        assert_eq!(state.count, 141);
+        assert_eq!(state.missing, 1);
+        assert!((state.cardinality.estimate() - 3.0).abs() < 1.0);
+        assert_eq!(
+            state.heavy_hitters.candidates().first().map(|item| item.0.as_str()),
+            Some("pune")
+        );
+    }
+
+    #[test]
+    fn categorical_states_merge_without_raw_strings() {
+        let mut left = CategoricalSketchState::default();
+        let mut right = CategoricalSketchState::default();
+        for _ in 0..60 {
+            left.observe_bytes(Some(b"alpha"));
+        }
+        for _ in 0..80 {
+            right.observe_bytes(Some(b"alpha"));
+        }
+        left.observe_bytes(Some(b"left"));
+        right.observe_bytes(Some(b"right"));
+        right.observe_missing();
+
+        let merged = left.merge(right);
+        assert_eq!(merged.count, 142);
+        assert_eq!(merged.missing, 1);
+        assert_eq!(
+            merged.heavy_hitters.candidates().first().map(|item| item.0.as_str()),
+            Some("alpha")
+        );
+    }
+
+    #[test]
+    fn labels_are_bounded_even_when_values_are_long() {
+        let mut sketch = StringHeavyHittersSketch::new(4, 8);
+        let value = b"abcdefghijklmnopqrstuvwxyz";
+        let mut hasher = StableByteHasher::new();
+        for byte in value {
+            hasher.update(*byte);
+        }
+        sketch.observe_hashed(hasher.finish(), value, 5);
+        assert_eq!(sketch.candidates()[0].0, "abcdefgh");
+        assert_eq!(sketch.max_label_bytes(), 8);
+    }
+}

From 60373643b7c200e313832db6b239b3671c26e874 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:38:33 +0530
Subject: [PATCH 173/439] feat(native): expose categorical sketch primitives

---
 rust/framevitals-core/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/rust/framevitals-core/src/lib.rs b/rust/framevitals-core/src/lib.rs
index eff6361..f0c3abf 100644
--- a/rust/framevitals-core/src/lib.rs
+++ b/rust/framevitals-core/src/lib.rs
@@ -7,6 +7,7 @@
 
 #[cfg(feature = "arrow")]
 pub mod arrow_scan;
+pub mod categorical_sketches;
 #[cfg(feature = "arrow")]
 pub mod fused_profile;
 pub mod sketches;

From cec80437688bba1cbc517831b4e7c2226e0eadb7 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:38:58 +0530
Subject: [PATCH 174/439] feat(native): consume Arrow UTF8 buffers without
 Python strings

---
 rust/framevitals-py/src/string_accumulator.rs | 205 ++++++++++++++++++
 1 file changed, 205 insertions(+)
 create mode 100644 rust/framevitals-py/src/string_accumulator.rs

diff --git a/rust/framevitals-py/src/string_accumulator.rs b/rust/framevitals-py/src/string_accumulator.rs
new file mode 100644
index 0000000..c2699bf
--- /dev/null
+++ b/rust/framevitals-py/src/string_accumulator.rs
@@ -0,0 +1,205 @@
+use framevitals_core::categorical_sketches::{CategoricalSketchState, StableByteHasher};
+use pyo3::buffer::{Element, PyBuffer, ReadOnlyCell};
+use pyo3::exceptions::PyBufferError;
+use pyo3::prelude::*;
+use pyo3::types::PyDict;
+
+trait OffsetValue: Element + Copy {
+    fn as_i64(self) -> i64;
+}
+
+impl OffsetValue for i32 {
+    fn as_i64(self) -> i64 {
+        i64::from(self)
+    }
+}
+
+impl OffsetValue for i64 {
+    fn as_i64(self) -> i64 {
+        self
+    }
+}
+
+fn is_valid(validity: Option<&[ReadOnlyCell<u8>]>, index: usize) -> bool {
+    let Some(validity) = validity else {
+        return true;
+    };
+    let byte_index = index / 8;
+    if byte_index >= validity.len() {
+        return false;
+    }
+    let bit = index % 8;
+    validity[byte_index].get() & (1_u8 << bit) != 0
+}
+
+fn update_utf8_buffers<T: OffsetValue>(
+    py: Python<'_>,
+    state: &mut CategoricalSketchState,
+    data: &Bound<'_, PyAny>,
+    offsets: &Bound<'_, PyAny>,
+    length: usize,
+    validity: Option<&Bound<'_, PyAny>>,
+    array_offset: usize,
+) -> PyResult<()> {
+    let data_buffer = PyBuffer::<u8>::get(data)?;
+    let offsets_buffer = PyBuffer::<T>::get(offsets)?;
+    let validity_buffer = validity.map(PyBuffer::<u8>::get).transpose()?;
+
+    if !data_buffer.is_c_contiguous() || !offsets_buffer.is_c_contiguous() {
+        return Err(PyBufferError::new_err(
+            "FrameVitals string kernels require contiguous Arrow data/offset buffers.",
+        ));
+    }
+    if let Some(buffer) = &validity_buffer {
+        if !buffer.is_c_contiguous() {
+            return Err(PyBufferError::new_err(
+                "FrameVitals string kernels require a contiguous validity bitmap.",
+            ));
+        }
+    }
+
+    let data_slice = data_buffer.as_slice(py).ok_or_else(|| {
+        PyBufferError::new_err("Could not borrow Arrow UTF8 data as a byte slice.")
+    })?;
+    let offsets_slice = offsets_buffer.as_slice(py).ok_or_else(|| {
+        PyBufferError::new_err("Could not borrow Arrow UTF8 offsets as a typed slice.")
+    })?;
+    let validity_slice = validity_buffer
+        .as_ref()
+        .and_then(|buffer| buffer.as_slice(py));
+
+    let required_offsets = array_offset
+        .checked_add(length)
+        .and_then(|value| value.checked_add(1))
+        .ok_or_else(|| PyBufferError::new_err("Arrow string offset range overflowed."))?;
+    if required_offsets > offsets_slice.len() {
+        return Err(PyBufferError::new_err(
+            "Arrow string offsets are shorter than the requested array range.",
+        ));
+    }
+
+    let label_limit = state.heavy_hitters.max_label_bytes();
+    for local_index in 0..length {
+        let logical_index = array_offset + local_index;
+        if !is_valid(validity_slice, logical_index) {
+            state.observe_missing();
+            continue;
+        }
+
+        let start = offsets_slice[logical_index].get().as_i64();
+        let end = offsets_slice[logical_index + 1].get().as_i64();
+        if start < 0 || end < start {
+            return Err(PyBufferError::new_err(
+                "Arrow string offsets contain an invalid range.",
+            ));
+        }
+        let start = usize::try_from(start)
+            .map_err(|_| PyBufferError::new_err("Arrow string offset is out of range."))?;
+        let end = usize::try_from(end)
+            .map_err(|_| PyBufferError::new_err("Arrow string offset is out of range."))?;
+        if end > data_slice.len() {
+            return Err(PyBufferError::new_err(
+                "Arrow string offset exceeds the UTF8 data buffer.",
+            ));
+        }
+
+        let mut hasher = StableByteHasher::new();
+        let mut label = Vec::with_capacity((end - start).min(label_limit));
+        for cell in &data_slice[start..end] {
+            let byte = cell.get();
+            hasher.update(byte);
+            if label.len() < label_limit {
+                label.push(byte);
+            }
+        }
+        state.observe_hashed(hasher.finish(), &label);
+    }
+    Ok(())
+}
+
+#[pyclass]
+pub(crate) struct StringAccumulator {
+    state: CategoricalSketchState,
+}
+
+#[pymethods]
+impl StringAccumulator {
+    #[new]
+    fn new() -> Self {
+        Self {
+            state: CategoricalSketchState::default(),
+        }
+    }
+
+    #[pyo3(signature = (data, offsets, length, validity=None, array_offset=0))]
+    fn update_utf8(
+        &mut self,
+        py: Python<'_>,
+        data: &Bound<'_, PyAny>,
+        offsets: &Bound<'_, PyAny>,
+        length: usize,
+        validity: Option<&Bound<'_, PyAny>>,
+        array_offset: usize,
+    ) -> PyResult<()> {
+        update_utf8_buffers::<i32>(
+            py,
+            &mut self.state,
+            data,
+            offsets,
+            length,
+            validity,
+            array_offset,
+        )
+    }
+
+    #[pyo3(signature = (data, offsets, length, validity=None, array_offset=0))]
+    fn update_large_utf8(
+        &mut self,
+        py: Python<'_>,
+        data: &Bound<'_, PyAny>,
+        offsets: &Bound<'_, PyAny>,
+        length: usize,
+        validity: Option<&Bound<'_, PyAny>>,
+        array_offset: usize,
+    ) -> PyResult<()> {
+        update_utf8_buffers::<i64>(
+            py,
+            &mut self.state,
+            data,
+            offsets,
+            length,
+            validity,
+            array_offset,
+        )
+    }
+
+    fn snapshot(&self, py: Python<'_>) -> PyResult<Py<PyDict>> {
+        let payload = PyDict::new(py);
+        payload.set_item("backend", "rust")?;
+        payload.set_item("count", self.state.count)?;
+        payload.set_item("missing", self.state.missing)?;
+        payload.set_item("observations", self.state.count + self.state.missing)?;
+        payload.set_item(
+            "cardinality_estimate",
+            self.state.cardinality.estimate().round() as u64,
+        )?;
+        payload.set_item("heavy_hitters", self.state.heavy_hitters.candidates())?;
+        payload.set_item("cardinality_method", "hyperloglog")?;
+        payload.set_item("heavy_hitter_method", "misra_gries_candidates")?;
+        payload.set_item("heavy_hitter_count_semantics", "lower_bound")?;
+        payload.set_item(
+            "max_retained_label_bytes",
+            self.state.heavy_hitters.max_label_bytes(),
+        )?;
+        Ok(payload.unbind())
+    }
+
+    fn reset(&mut self) {
+        self.state = CategoricalSketchState::default();
+    }
+
+    #[getter]
+    fn observations(&self) -> u64 {
+        self.state.count + self.state.missing
+    }
+}

From f43e9dc207d4acfd75db875c2de130c7558cec89 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:39:23 +0530
Subject: [PATCH 175/439] feat(native): expose Arrow string accumulator to
 Python

---
 rust/framevitals-py/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/rust/framevitals-py/src/lib.rs b/rust/framevitals-py/src/lib.rs
index 9baec24..77401c6 100644
--- a/rust/framevitals-py/src/lib.rs
+++ b/rust/framevitals-py/src/lib.rs
@@ -1,11 +1,14 @@
 //! Python bindings for FrameVitals native kernels.
 
+mod string_accumulator;
+
 use framevitals_core::sketches::NumericSketchState;
 use framevitals_core::NumericState;
 use pyo3::buffer::PyBuffer;
 use pyo3::exceptions::PyBufferError;
 use pyo3::prelude::*;
 use pyo3::types::PyDict;
+use string_accumulator::StringAccumulator;
 
 fn exact_state_dict<'py>(py: Python<'py>, state: &NumericState) -> PyResult<Bound<'py, PyDict>> {
     let payload = PyDict::new(py);
@@ -200,6 +203,7 @@ fn backend_info() -> (&'static str, &'static str) {
 #[pymodule]
 fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_class::<NumericAccumulator>()?;
+    module.add_class::<StringAccumulator>()?;
     module.add_function(wrap_pyfunction!(numeric_state_f64, module)?)?;
     module.add_function(wrap_pyfunction!(numeric_profile_f64, module)?)?;
     module.add_function(wrap_pyfunction!(backend_info, module)?)?;

From 995735571937e70d5bef5369f4cbf3d607e24cdb Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:39:47 +0530
Subject: [PATCH 176/439] feat: expose native string accumulator

---
 src/framevitals/backends.py | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/src/framevitals/backends.py b/src/framevitals/backends.py
index 7398aff..e433fbd 100644
--- a/src/framevitals/backends.py
+++ b/src/framevitals/backends.py
@@ -1,8 +1,8 @@
-"""Backend routing for FrameVitals numeric kernels.
+"""Backend routing for FrameVitals native and NumPy kernels.
 
 The router keeps optional native/GPU dependencies behind lazy imports. Normal
 installations remain NumPy-first, while a compiled ``framevitals._native``
-extension can transparently accelerate compatible numeric scans.
+extension can transparently accelerate compatible streaming scans.
 """
 
 from __future__ import annotations
@@ -26,7 +26,7 @@ def native_available() -> bool:
 
 
 def resolve_numeric_backend(requested: str | None = None) -> Literal["numpy", "rust"]:
-    """Resolve the backend for exact/streaming numeric primitives.
+    """Resolve the backend for exact/streaming primitives.
 
     ``FRAMEVITALS_BACKEND`` may set ``auto``, ``numpy`` or ``rust``. Explicit
     function arguments take precedence. Requesting Rust without the extension
@@ -117,12 +117,7 @@ def numeric_profile(
     backend: BackendName | str | None = None,
     stream_id: int = 0,
 ) -> dict[str, Any]:
-    """Run the fused native profile when available.
-
-    The NumPy fallback intentionally returns only exact state for now rather
-    than pretending to reproduce native approximate sketches with a different
-    algorithm. ``sketches_available`` makes that distinction explicit.
-    """
+    """Run the fused native numeric profile when available."""
     array = _float64_array(values)
     selected = resolve_numeric_backend(backend)
     if selected == "rust":
@@ -137,18 +132,22 @@ def numeric_profile(
 
 
 def create_numeric_accumulator(*, stream_id: int = 0):
-    """Create a persistent native accumulator for multi-batch scans.
-
-    Returns ``None`` when the active backend is NumPy. This keeps callers free
-    to implement a mergeable Python fallback without importing the native
-    extension unless it is actually selected.
-    """
+    """Create a persistent native numeric accumulator for multi-batch scans."""
     if resolve_numeric_backend() != "rust":
         return None
     native = import_module("framevitals._native")
     return native.NumericAccumulator(stream_id=int(stream_id))
 
 
+def create_string_accumulator():
+    """Create a persistent native UTF-8 sketch accumulator when available."""
+    if resolve_numeric_backend() != "rust":
+        return None
+    native = import_module("framevitals._native")
+    accumulator = getattr(native, "StringAccumulator", None)
+    return accumulator() if accumulator is not None else None
+
+
 def backend_status() -> dict[str, Any]:
     """Return lightweight backend availability without importing native code."""
     has_native = native_available()

From e616150e677fd3cb6378ce1a272e817275df1126 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:40:42 +0530
Subject: [PATCH 177/439] feat: profile streaming UTF8 categories with native
 sketches

---
 src/framevitals/streaming_profile.py | 131 +++++++++++++++++++++++----
 1 file changed, 112 insertions(+), 19 deletions(-)

diff --git a/src/framevitals/streaming_profile.py b/src/framevitals/streaming_profile.py
index f676f03..7314d88 100644
--- a/src/framevitals/streaming_profile.py
+++ b/src/framevitals/streaming_profile.py
@@ -2,7 +2,8 @@
 
 The streaming path keeps full-row work to mergeable column state and retains
 only a bounded, evenly spaced row sample for analyses that still require row
-relationships (categorical top values, duplicate estimation, and correlation).
+relationships (duplicate estimation and correlation). Native builds can also
+consume Arrow UTF-8 buffers directly for full-file categorical sketches.
 """
 
 from __future__ import annotations
@@ -17,6 +18,7 @@
 from framevitals.analysis_state import NumericColumnState
 from framevitals.backends import (
     create_numeric_accumulator,
+    create_string_accumulator,
     numeric_state,
     resolve_numeric_backend,
 )
@@ -71,6 +73,15 @@ def _column_groups(schema) -> tuple[list[str], list[str], list[str]]:
     return numeric, categorical, dates
 
 
+def _string_columns(schema) -> list[str]:
+    pa = _require_pyarrow()
+    return [
+        field.name
+        for field in schema
+        if pa.types.is_string(field.type) or pa.types.is_large_string(field.type)
+    ]
+
+
 def numeric_columns_for_streaming_source(source: StreamingDatasetSource) -> list[str]:
     """Return Arrow numeric column names without reading row data."""
     schema_method = getattr(source, "schema", None)
@@ -94,6 +105,41 @@ def _arrow_numeric_to_float64(array) -> np.ndarray:
     return np.ascontiguousarray(converted, dtype=np.float64)
 
 
+def _update_native_string_accumulator(accumulator, array) -> None:
+    """Feed Arrow UTF-8 buffers to Rust without constructing Python strings."""
+    pa = _require_pyarrow()
+    validity_buffer, offsets_buffer, data_buffer = array.buffers()
+    validity = (
+        np.frombuffer(validity_buffer, dtype=np.uint8)
+        if validity_buffer is not None
+        else None
+    )
+    data = (
+        np.frombuffer(data_buffer, dtype=np.uint8)
+        if data_buffer is not None
+        else np.empty(0, dtype=np.uint8)
+    )
+
+    if pa.types.is_large_string(array.type):
+        offsets = np.frombuffer(offsets_buffer, dtype=np.int64)
+        accumulator.update_large_utf8(
+            data,
+            offsets,
+            len(array),
+            validity,
+            int(array.offset),
+        )
+    else:
+        offsets = np.frombuffer(offsets_buffer, dtype=np.int32)
+        accumulator.update_utf8(
+            data,
+            offsets,
+            len(array),
+            validity,
+            int(array.offset),
+        )
+
+
 def _state_from_payload(payload: dict[str, Any]) -> NumericColumnState:
     count = int(payload["count"])
     variance = payload.get("variance")
@@ -265,12 +311,7 @@ def build_streaming_profile(
     sample_rows: int = STREAM_SAMPLE_ROWS,
     return_sample: bool = False,
 ) -> dict[str, Any] | tuple[dict[str, Any], pd.DataFrame]:
-    """Profile a streaming Arrow source without materializing all rows.
-
-    ``return_sample=True`` lets planning/orchestration reuse the bounded row
-    sample generated during the same full-file scan instead of reading the
-    source a second time.
-    """
+    """Profile a streaming Arrow source without materializing all rows."""
     if batch_size < 1:
         raise ValueError("batch_size must be at least 1.")
     if sample_rows < 1:
@@ -289,6 +330,7 @@ def build_streaming_profile(
     schema = schema_method()
     columns = [field.name for field in schema]
     numeric_cols, categorical_cols, date_cols = _column_groups(schema)
+    string_cols = _string_columns(schema)
     dtypes = {field.name: str(field.type) for field in schema}
 
     positions = _sample_positions(int(rows), int(sample_rows))
@@ -298,12 +340,18 @@ def build_streaming_profile(
     selected_backend = resolve_numeric_backend()
 
     native_accumulators: dict[str, Any] = {}
+    native_string_accumulators: dict[str, Any] = {}
     python_states = {column: NumericColumnState() for column in numeric_cols}
     if selected_backend == "rust":
         native_accumulators = {
             column: create_numeric_accumulator(stream_id=index)
             for index, column in enumerate(numeric_cols)
         }
+        native_string_accumulators = {
+            column: accumulator
+            for column in string_cols
+            if (accumulator := create_string_accumulator()) is not None
+        }
 
     offset = 0
     batches_scanned = 0
@@ -320,6 +368,9 @@ def build_streaming_profile(
             array = batch.column(index)
             if column not in numeric_cols:
                 missing_counts[column] += int(array.null_count)
+                string_accumulator = native_string_accumulators.get(column)
+                if string_accumulator is not None:
+                    _update_native_string_accumulator(string_accumulator, array)
                 continue
 
             values = _arrow_numeric_to_float64(array)
@@ -327,9 +378,9 @@ def build_streaming_profile(
             if accumulator is not None:
                 accumulator.update_f64(values)
             else:
-                payload = numeric_state(values, backend="numpy")
+                numeric_payload = numeric_state(values, backend="numpy")
                 python_states[column] = python_states[column].merge(
-                    _state_from_payload(payload)
+                    _state_from_payload(numeric_payload)
                 )
 
         offset += int(batch.num_rows)
@@ -351,10 +402,10 @@ def build_streaming_profile(
     quantile_accuracy = None
     if selected_backend == "rust":
         for column in numeric_cols:
-            payload = dict(native_accumulators[column].snapshot())
-            numeric_summary[column] = _summary_from_native_snapshot(payload)
-            missing_counts[column] = int(payload["missing"])
-            quantile_accuracy = payload.get("quantiles", {}).get(
+            numeric_payload = dict(native_accumulators[column].snapshot())
+            numeric_summary[column] = _summary_from_native_snapshot(numeric_payload)
+            missing_counts[column] = int(numeric_payload["missing"])
+            quantile_accuracy = numeric_payload.get("quantiles", {}).get(
                 "relative_accuracy",
                 quantile_accuracy,
             )
@@ -385,25 +436,66 @@ def build_streaming_profile(
             "raw_observations_retained": False,
         }
 
-    missing_series = pd.Series(missing_counts, dtype="int64")
-    missing_percent = (missing_series / max(rows, 1) * 100).round(2)
-
-    categorical_summary = {}
+    categorical_summary: dict[str, dict[str, Any]] = {}
+    native_categorical_columns: list[str] = []
+    sample_categorical_columns: list[str] = []
     for column in categorical_cols:
+        string_accumulator = native_string_accumulators.get(column)
+        if string_accumulator is not None:
+            categorical_payload = dict(string_accumulator.snapshot())
+            missing_counts[column] = int(categorical_payload["missing"])
+            count = int(categorical_payload["count"])
+            estimate = min(int(categorical_payload["cardinality_estimate"]), count)
+            categorical_summary[column] = {
+                "unique_values": estimate,
+                "top_values": {
+                    str(label): int(candidate_count)
+                    for label, candidate_count in categorical_payload["heavy_hitters"][:10]
+                },
+                "approximate": True,
+                "unique_values_method": categorical_payload["cardinality_method"],
+                "top_values_method": categorical_payload["heavy_hitter_method"],
+                "top_values_count_semantics": categorical_payload[
+                    "heavy_hitter_count_semantics"
+                ],
+            }
+            native_categorical_columns.append(column)
+            continue
+
         if column not in sample:
             continue
         counts = sample[column].value_counts(dropna=False).head(10)
         categorical_summary[column] = {
             "unique_values": int(sample[column].nunique(dropna=True)),
             "top_values": {str(key): int(value) for key, value in counts.items()},
+            "approximate": len(sample) < rows,
         }
+        sample_categorical_columns.append(column)
+
+    if native_categorical_columns:
+        categorical_method = (
+            "native_full_stream_sketch"
+            if not sample_categorical_columns
+            else "native_full_stream_sketch_with_sample_fallback"
+        )
+    else:
+        categorical_method = "exact" if len(sample) == rows else "evenly_spaced_row_sample"
     categorical_metadata = {
-        "method": "exact" if len(sample) == rows else "evenly_spaced_row_sample",
-        "sampled": len(sample) < rows,
+        "method": categorical_method,
+        "sampled": bool(sample_categorical_columns and len(sample) < rows),
         "sample_rows": int(len(sample)),
         "source_rows": int(rows),
+        "native_full_stream_columns": native_categorical_columns,
+        "sample_fallback_columns": sample_categorical_columns,
+        "cardinality_method": "hyperloglog" if native_categorical_columns else None,
+        "top_values_count_semantics": (
+            "lower_bound_candidates" if native_categorical_columns else None
+        ),
     }
 
+    missing_series = pd.Series(missing_counts, dtype="int64")
+    missing_percent = (missing_series / max(rows, 1) * 100).round(2)
+
     duplicate_rows, duplicate_metadata = _duplicate_summary(sample, int(rows))
     correlations, correlation_metadata = _bounded_correlations(sample, numeric_cols)
     correlation_metadata.update({
@@ -448,6 +540,7 @@ def build_streaming_profile(
             "sample_rows": int(len(sample)),
             "sample_strategy": "evenly_spaced_global_rows",
             "numeric_backend": selected_backend,
+            "native_string_sketch_columns": native_categorical_columns,
         },
     }
     if return_sample:

From 9380d304bc627031ceb879f8ab9567a7571355c1 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:42:35 +0530
Subject: [PATCH 178/439] style(native): apply rustfmt to categorical sketches

---
 .../src/categorical_sketches.rs                 | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/rust/framevitals-core/src/categorical_sketches.rs b/rust/framevitals-core/src/categorical_sketches.rs
index cb34755..c72c1cc 100644
--- a/rust/framevitals-core/src/categorical_sketches.rs
+++ b/rust/framevitals-core/src/categorical_sketches.rs
@@ -114,7 +114,10 @@ impl StringHeavyHittersSketch {
 
     #[must_use]
     pub fn merge(mut self, other: Self) -> Self {
-        assert_eq!(self.capacity, other.capacity, "heavy-hitter capacity mismatch");
+        assert_eq!(
+            self.capacity, other.capacity,
+            "heavy-hitter capacity mismatch"
+        );
         assert_eq!(
             self.max_label_bytes, other.max_label_bytes,
             "heavy-hitter label limit mismatch"
@@ -246,7 +249,11 @@ mod tests {
         assert_eq!(state.missing, 1);
         assert!((state.cardinality.estimate() - 3.0).abs() < 1.0);
         assert_eq!(
-            state.heavy_hitters.candidates().first().map(|item| item.0.as_str()),
+            state
+                .heavy_hitters
+                .candidates()
+                .first()
+                .map(|item| item.0.as_str()),
             Some("pune")
         );
     }
@@ -269,7 +276,11 @@ mod tests {
         assert_eq!(merged.count, 142);
         assert_eq!(merged.missing, 1);
         assert_eq!(
-            merged.heavy_hitters.candidates().first().map(|item| item.0.as_str()),
+            merged
+                .heavy_hitters
+                .candidates()
+                .first()
+                .map(|item| item.0.as_str()),
             Some("alpha")
         );
     }

From c82cca106a2f4f8391231f8b46be074960b51df4 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:43:43 +0530
Subject: [PATCH 179/439] fix(native): derive categorical sketch defaults

---
 rust/framevitals-core/src/categorical_sketches.rs | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/rust/framevitals-core/src/categorical_sketches.rs b/rust/framevitals-core/src/categorical_sketches.rs
index c72c1cc..9f31f24 100644
--- a/rust/framevitals-core/src/categorical_sketches.rs
+++ b/rust/framevitals-core/src/categorical_sketches.rs
@@ -161,7 +161,7 @@ impl Default for StringHeavyHittersSketch {
     }
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Default)]
 pub struct CategoricalSketchState {
     pub count: u64,
     pub missing: u64,
@@ -169,17 +169,6 @@ pub struct CategoricalSketchState {
     pub heavy_hitters: StringHeavyHittersSketch,
 }
 
-impl Default for CategoricalSketchState {
-    fn default() -> Self {
-        Self {
-            count: 0,
-            missing: 0,
-            cardinality: HyperLogLog::default(),
-            heavy_hitters: StringHeavyHittersSketch::default(),
-        }
-    }
-}
-
 impl CategoricalSketchState {
     pub fn observe_missing(&mut self) {
         self.missing = self.missing.saturating_add(1);

From 8fb5a30752e4be1dafb4be0b5798ba3ede9a7365 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:59:10 +0530
Subject: [PATCH 180/439] feat: add streaming-aware health scoring

---
 src/framevitals/health_score.py | 215 +++++++++++++++++++++++++-------
 1 file changed, 173 insertions(+), 42 deletions(-)

diff --git a/src/framevitals/health_score.py b/src/framevitals/health_score.py
index fc298be..aee9d7d 100644
--- a/src/framevitals/health_score.py
+++ b/src/framevitals/health_score.py
@@ -1,7 +1,12 @@
-import pandas as pd
+from __future__ import annotations
+
+from typing import Any, Mapping
+
 import numpy as np
+import pandas as pd
 
-def calculate_outlier_percent(df):
+
+def calculate_outlier_percent(df: pd.DataFrame):
     numeric_cols = df.select_dtypes(include=[np.number]).columns
     if len(numeric_cols) == 0:
         return 0.0, {}
@@ -28,36 +33,40 @@ def calculate_outlier_percent(df):
     percent = round(outlier_count / max(total_cells, 1) * 100, 2)
     return percent, details
 
-def calculate_health_score(df, profile):
-    rows = max(profile["shape"]["rows"], 1)
-    columns = max(profile["shape"]["columns"], 1)
-
-    missing_total = sum(
-        value for value in profile["missing_counts"].values()
-        if isinstance(value, int)
-    )
-
-    missing_percent = missing_total / (rows * columns) * 100
-    duplicate_percent = profile["duplicate_percent"]
-    outlier_percent, outlier_details = calculate_outlier_percent(df)
-
-    constant_columns = []
-    high_cardinality_columns = []
-
-    for col in df.columns:
-        unique_count = df[col].nunique(dropna=True)
-        if unique_count <= 1:
-            constant_columns.append(col)
-        if df[col].dtype == "object" and unique_count > 0.8 * rows:
-            high_cardinality_columns.append(col)
 
-    constant_percent = len(constant_columns) / columns * 100
-    high_cardinality_percent = len(high_cardinality_columns) / columns * 100
+def _health_label(overall: float) -> str:
+    if overall >= 90:
+        return "Excellent"
+    if overall >= 75:
+        return "Good"
+    if overall >= 60:
+        return "Moderate"
+    if overall >= 40:
+        return "Poor"
+    return "Critical"
+
+
+def _score_payload(
+    *,
+    missing_percent: float,
+    duplicate_percent: float,
+    outlier_percent: float,
+    constant_columns: list[str],
+    high_cardinality_columns: list[str],
+    columns: int,
+    outlier_details: Mapping[str, Any],
+    execution: Mapping[str, Any] | None = None,
+) -> dict[str, Any]:
+    constant_percent = len(constant_columns) / max(columns, 1) * 100
+    high_cardinality_percent = len(high_cardinality_columns) / max(columns, 1) * 100
 
     completeness_score = max(0, round(100 - missing_percent, 2))
     uniqueness_score = max(0, round(100 - duplicate_percent, 2))
     outlier_safety_score = max(0, round(100 - outlier_percent, 2))
-    consistency_score = max(0, round(100 - constant_percent - high_cardinality_percent, 2))
+    consistency_score = max(
+        0,
+        round(100 - constant_percent - high_cardinality_percent, 2),
+    )
 
     overall = round(
         completeness_score * 0.30
@@ -67,23 +76,11 @@ def calculate_health_score(df, profile):
         + 10,
         2,
     )
-
     overall = max(0, min(100, overall))
 
-    if overall >= 90:
-        label = "Excellent"
-    elif overall >= 75:
-        label = "Good"
-    elif overall >= 60:
-        label = "Moderate"
-    elif overall >= 40:
-        label = "Poor"
-    else:
-        label = "Critical"
-
-    return {
+    payload: dict[str, Any] = {
         "overall_score": overall,
-        "label": label,
+        "label": _health_label(overall),
         "components": {
             "completeness": completeness_score,
             "consistency": consistency_score,
@@ -96,6 +93,140 @@ def calculate_health_score(df, profile):
             "outlier_percent": outlier_percent,
             "constant_columns": constant_columns,
             "high_cardinality_columns": high_cardinality_columns,
-            "outlier_details": outlier_details,
+            "outlier_details": dict(outlier_details),
         },
     }
+    if execution is not None:
+        payload["execution"] = dict(execution)
+    return payload
+
+
+def _profile_constant_and_cardinality_columns(
+    profile: Mapping[str, Any],
+) -> tuple[list[str], list[str]]:
+    """Infer consistency signals from a profile without raw full-row access."""
+    rows = int(profile.get("shape", {}).get("rows", 0) or 0)
+    numeric_summary = profile.get("numeric_summary", {})
+    categorical_summary = profile.get("categorical_summary", {})
+
+    constant_columns: list[str] = []
+    high_cardinality_columns: list[str] = []
+
+    if isinstance(numeric_summary, Mapping):
+        for column, raw_summary in numeric_summary.items():
+            if not isinstance(raw_summary, Mapping):
+                continue
+            count = int(raw_summary.get("count", 0) or 0)
+            minimum = raw_summary.get("min")
+            maximum = raw_summary.get("max")
+            if count > 0 and minimum is not None and maximum is not None and minimum == maximum:
+                constant_columns.append(str(column))
+
+    if isinstance(categorical_summary, Mapping):
+        for column, raw_summary in categorical_summary.items():
+            if not isinstance(raw_summary, Mapping):
+                continue
+            unique_values = int(raw_summary.get("unique_values", 0) or 0)
+            if unique_values <= 1:
+                constant_columns.append(str(column))
+            if rows > 0 and unique_values > 0.8 * rows:
+                high_cardinality_columns.append(str(column))
+
+    return sorted(set(constant_columns)), sorted(set(high_cardinality_columns))
+
+
+def calculate_health_score_from_profile_sample(
+    profile: Mapping[str, Any],
+    sample: pd.DataFrame,
+) -> dict[str, Any]:
+    """Calculate a bounded health score from a full profile plus row sample.
+
+    This is intended for streaming sources. Completeness and duplicate metrics
+    come from the full streaming profile, while outlier safety is estimated from
+    the bounded sample. Constant/high-cardinality signals use full-stream profile
+    summaries when available.
+    """
+    rows = max(int(profile.get("shape", {}).get("rows", 0) or 0), 1)
+    columns = max(int(profile.get("shape", {}).get("columns", 0) or 0), 1)
+    missing_counts = profile.get("missing_counts", {})
+    missing_total = sum(
+        int(value)
+        for value in missing_counts.values()
+        if isinstance(value, (int, np.integer))
+    ) if isinstance(missing_counts, Mapping) else 0
+    missing_percent = missing_total / (rows * columns) * 100
+    duplicate_percent = float(profile.get("duplicate_percent", 0.0) or 0.0)
+
+    outlier_percent, sample_outlier_details = calculate_outlier_percent(sample)
+    sample_rows = len(sample)
+    outlier_scale = rows / max(sample_rows, 1)
+    outlier_details = {
+        column: int(round(count * outlier_scale))
+        for column, count in sample_outlier_details.items()
+    }
+    constant_columns, high_cardinality_columns = _profile_constant_and_cardinality_columns(
+        profile
+    )
+
+    duplicate_metadata = profile.get("duplicate_metadata", {})
+    duplicate_estimated = bool(
+        isinstance(duplicate_metadata, Mapping) and duplicate_metadata.get("sampled")
+    )
+
+    return _score_payload(
+        missing_percent=missing_percent,
+        duplicate_percent=duplicate_percent,
+        outlier_percent=outlier_percent,
+        constant_columns=constant_columns,
+        high_cardinality_columns=high_cardinality_columns,
+        columns=columns,
+        outlier_details=outlier_details,
+        execution={
+            "method": "streaming_profile_with_bounded_row_sample",
+            "source_rows": rows,
+            "sample_rows": sample_rows,
+            "full_materialization": False,
+            "components": {
+                "completeness": "full_stream_exact",
+                "uniqueness": (
+                    "full_stream_sample_estimate" if duplicate_estimated else "full_stream_exact"
+                ),
+                "consistency": "full_stream_profile",
+                "outlier_safety": (
+                    "bounded_row_sample_estimate" if sample_rows < rows else "exact"
+                ),
+            },
+        },
+    )
+
+
+def calculate_health_score(df: pd.DataFrame, profile: Mapping[str, Any]):
+    rows = max(int(profile["shape"]["rows"]), 1)
+    columns = max(int(profile["shape"]["columns"]), 1)
+
+    missing_total = sum(
+        value for value in profile["missing_counts"].values() if isinstance(value, int)
+    )
+    missing_percent = missing_total / (rows * columns) * 100
+    duplicate_percent = profile["duplicate_percent"]
+    outlier_percent, outlier_details = calculate_outlier_percent(df)
+
+    constant_columns = []
+    high_cardinality_columns = []
+
+    for col in df.columns:
+        unique_count = df[col].nunique(dropna=True)
+        if unique_count <= 1:
+            constant_columns.append(col)
+        if df[col].dtype == "object" and unique_count > 0.8 * rows:
+            high_cardinality_columns.append(col)
+
+    return _score_payload(
+        missing_percent=missing_percent,
+        duplicate_percent=duplicate_percent,
+        outlier_percent=outlier_percent,
+        constant_columns=constant_columns,
+        high_cardinality_columns=high_cardinality_columns,
+        columns=columns,
+        outlier_details=outlier_details,
+    )

From cb3021a32ddda84957361155e2efb7f8f242ffbd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 12:59:28 +0530
Subject: [PATCH 181/439] feat: stream health checks for Arrow sources

---
 src/framevitals/focused.py | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index 633112b..84525c5 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -64,12 +64,22 @@ def roles(data: DataInput) -> dict[str, Any]:
 
 
 def health(data: DataInput) -> dict[str, Any]:
+    source = resolve_source(data)
+    metadata = source.inspect()
+    if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
+        from framevitals.health_score import calculate_health_score_from_profile_sample
+        from framevitals.streaming_profile import build_streaming_profile
+
+        dataset_profile, sample = build_streaming_profile(source, return_sample=True)
+        payload = calculate_health_score_from_profile_sample(dataset_profile, sample)
+        return _named(payload, metadata.name)
+
     from framevitals.health_score import calculate_health_score
     from framevitals.profiler import build_profile
 
-    dataframe, source_name = _load(data)
+    dataframe = source.load()
     dataset_profile = build_profile(dataframe)
-    return _named(calculate_health_score(dataframe, dataset_profile), source_name)
+    return _named(calculate_health_score(dataframe, dataset_profile), metadata.name)
 
 
 def ml_readiness(data: DataInput) -> dict[str, Any]:
@@ -183,29 +193,23 @@ def relationships(
         )
 
         numeric_columns = numeric_columns_for_streaming_source(source)
-        dataframe = sample_streaming_source(
+        sample = sample_streaming_source(
             source,
             sample_rows=max_sample_rows,
             columns=numeric_columns,
         )
         payload = build_numeric_relationship_graph(
-            dataframe,
+            sample,
             max_sample_rows=max_sample_rows,
             projections=projections,
             min_abs_correlation=min_abs_correlation,
             max_candidate_pairs=max_candidate_pairs,
             max_edges_returned=max_edges_returned,
         )
-        if payload.get("available"):
-            sample = payload.setdefault("sample", {})
-            sample.update({
-                "source_rows": int(metadata.rows or len(dataframe)),
-                "sample_rows": int(len(dataframe)),
-                "sampled": int(metadata.rows or len(dataframe)) > len(dataframe),
-                "strategy": "streaming_evenly_spaced_global_rows",
-                "full_materialization": False,
-            })
-            payload["source"] = metadata.to_dict()
+        payload["source_rows"] = metadata.rows
+        payload["source_columns"] = metadata.columns
+        payload["streaming_source"] = True
+        payload["full_materialization"] = False
         return _named(payload, metadata.name)
 
     dataframe = source.load()

From 36edaeca90161ebad91525d89393872ce4a6464c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:00:54 +0530
Subject: [PATCH 182/439] test: verify streaming parquet health path

---
 tests/test_parquet_streaming.py | 42 +++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
index 4b79e96..68df628 100644
--- a/tests/test_parquet_streaming.py
+++ b/tests/test_parquet_streaming.py
@@ -84,6 +84,48 @@ def test_large_parquet_retains_only_bounded_row_sample(tmp_path):
     assert result["missing_counts"]["value"] == int(frame["value"].isna().sum())
 
 
+def test_public_health_streams_parquet_without_calling_load(tmp_path, monkeypatch):
+    path = tmp_path / "health.parquet"
+    frame = _write_parquet(path, rows=12_000)
+
+    def fail_load(self):
+        raise AssertionError("health must not materialize the complete Parquet file")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.health(path)
+
+    assert result["dataset_name"] == "health.parquet"
+    assert 0 <= result["overall_score"] <= 100
+    assert result["execution"]["method"] == "streaming_profile_with_bounded_row_sample"
+    assert result["execution"]["full_materialization"] is False
+    assert result["execution"]["source_rows"] == len(frame)
+    assert result["execution"]["sample_rows"] == len(frame)
+    assert result["execution"]["components"]["completeness"] == "full_stream_exact"
+    assert result["execution"]["components"]["outlier_safety"] == "exact"
+    expected_missing = int(frame["value"].isna().sum()) / (len(frame) * 4) * 100
+    assert result["details"]["missing_percent"] == pytest.approx(
+        round(expected_missing, 2)
+    )
+
+
+def test_large_public_health_discloses_bounded_outlier_estimate(tmp_path, monkeypatch):
+    path = tmp_path / "large-health.parquet"
+    frame = _write_parquet(path, rows=60_000)
+
+    def fail_load(self):
+        raise AssertionError("large health must stay on the streaming path")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.health(path)
+
+    execution = result["execution"]
+    assert execution["full_materialization"] is False
+    assert execution["source_rows"] == len(frame)
+    assert execution["sample_rows"] == 50_000
+    assert execution["components"]["outlier_safety"] == "bounded_row_sample_estimate"
+    assert execution["components"]["uniqueness"] == "full_stream_sample_estimate"
+
+
 def test_plan_reads_only_bounded_parquet_sample_and_uses_true_shape(tmp_path, monkeypatch):
     path = tmp_path / "plan.parquet"
     frame = _write_parquet(path, rows=12_000)

From 961303fa0e21865096bdd47a9e22ce87735cdc4b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:01:36 +0530
Subject: [PATCH 183/439] feat: score ML readiness from profiles

---
 src/framevitals/ml_readiness.py | 97 ++++++++++++++++++++++++---------
 1 file changed, 70 insertions(+), 27 deletions(-)

diff --git a/src/framevitals/ml_readiness.py b/src/framevitals/ml_readiness.py
index 36dc3ed..8b2ddb0 100644
--- a/src/framevitals/ml_readiness.py
+++ b/src/framevitals/ml_readiness.py
@@ -12,33 +12,16 @@
 _CATEGORICAL_DTYPES = ["object", "string", "category", "bool"]
 
 
-def calculate_ml_readiness(df, profile=None):
-    """Calculate ML-readiness while reusing profile metrics when available.
-
-    ``profile`` is optional to preserve the standalone helper API. The main
-    pipeline passes the already-built profile so FrameVitals does not rescan
-    the full dataset for missing values, duplicates, and basic column groups.
-    """
-    rows, columns = df.shape
-
-    if profile is None:
-        # Standalone compatibility path. The public focused API builds a profile
-        # first and therefore avoids repeating these scans.
-        missing_total = sum(int(df[column].isna().sum()) for column in df.columns)
-        missing_percent = float(missing_total / max(rows * columns, 1) * 100)
-        duplicate_percent = float(df.duplicated().sum() / max(rows, 1) * 100)
-        categorical_cols = df.select_dtypes(include=_CATEGORICAL_DTYPES).columns.tolist()
-        numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
-    else:
-        missing_total = sum(
-            int(value)
-            for value in profile.get("missing_counts", {}).values()
-            if value is not None
-        )
-        missing_percent = float(missing_total / max(rows * columns, 1) * 100)
-        duplicate_percent = float(profile.get("duplicate_percent", 0.0))
-        categorical_cols = list(profile.get("categorical_columns", []))
-        numeric_cols = list(profile.get("numeric_columns", []))
+def _score_ml_readiness(
+    *,
+    rows: int,
+    columns: int,
+    missing_total: int,
+    duplicate_percent: float,
+    categorical_cols: list[str],
+    numeric_cols: list[str],
+) -> dict[str, Any]:
+    missing_percent = float(missing_total / max(rows * columns, 1) * 100)
 
     encoding_penalty = min(20, len(categorical_cols) * 2)
     missing_penalty = min(30, missing_percent)
@@ -75,6 +58,66 @@ def calculate_ml_readiness(df, profile=None):
     }
 
 
+def calculate_ml_readiness_from_profile(profile: dict[str, Any]) -> dict[str, Any]:
+    """Calculate ML-readiness directly from a completed dataset profile."""
+    shape = profile.get("shape", {})
+    rows = int(shape.get("rows", 0) or 0)
+    columns = int(shape.get("columns", len(profile.get("columns", []))) or 0)
+    missing_total = sum(
+        int(value)
+        for value in profile.get("missing_counts", {}).values()
+        if value is not None
+    )
+    duplicate_percent = float(profile.get("duplicate_percent", 0.0))
+    categorical_cols = list(profile.get("categorical_columns", []))
+    numeric_cols = list(profile.get("numeric_columns", []))
+
+    return _score_ml_readiness(
+        rows=rows,
+        columns=columns,
+        missing_total=missing_total,
+        duplicate_percent=duplicate_percent,
+        categorical_cols=categorical_cols,
+        numeric_cols=numeric_cols,
+    )
+
+
+def calculate_ml_readiness(df, profile=None):
+    """Calculate ML-readiness while reusing profile metrics when available.
+
+    ``profile`` is optional to preserve the standalone helper API. The main
+    pipeline passes the already-built profile so FrameVitals does not rescan
+    the full dataset for missing values, duplicates, and basic column groups.
+    """
+    rows, columns = df.shape
+
+    if profile is None:
+        # Standalone compatibility path. The public focused API builds a profile
+        # first and therefore avoids repeating these scans.
+        missing_total = sum(int(df[column].isna().sum()) for column in df.columns)
+        duplicate_percent = float(df.duplicated().sum() / max(rows, 1) * 100)
+        categorical_cols = df.select_dtypes(include=_CATEGORICAL_DTYPES).columns.tolist()
+        numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    else:
+        missing_total = sum(
+            int(value)
+            for value in profile.get("missing_counts", {}).values()
+            if value is not None
+        )
+        duplicate_percent = float(profile.get("duplicate_percent", 0.0))
+        categorical_cols = list(profile.get("categorical_columns", []))
+        numeric_cols = list(profile.get("numeric_columns", []))
+
+    return _score_ml_readiness(
+        rows=int(rows),
+        columns=int(columns),
+        missing_total=missing_total,
+        duplicate_percent=duplicate_percent,
+        categorical_cols=categorical_cols,
+        numeric_cols=numeric_cols,
+    )
+
+
 class _CallableReadinessModule(ModuleType):
     """Preserve ``fv.ml_readiness(data)`` after importing this compatibility module.
 

From 86fdab744cf3e236f8f5c7355f470575d1c00dc3 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:01:56 +0530
Subject: [PATCH 184/439] feat: add bounded streaming quality diagnostics

---
 src/framevitals/streaming_quality.py | 176 +++++++++++++++++++++++++++
 1 file changed, 176 insertions(+)
 create mode 100644 src/framevitals/streaming_quality.py

diff --git a/src/framevitals/streaming_quality.py b/src/framevitals/streaming_quality.py
new file mode 100644
index 0000000..662cea0
--- /dev/null
+++ b/src/framevitals/streaming_quality.py
@@ -0,0 +1,176 @@
+"""Quality diagnostics for streaming dataset sources.
+
+This adapter reuses FrameVitals' deterministic quality checks on a bounded row
+sample while preserving full-source profile facts. Findings whose truth cannot
+be proven from a sample (for example primary-key uniqueness or duplicate-column
+identity) are explicitly reported as candidates rather than full-source facts.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+import pandas as pd
+
+from framevitals.column_roles import infer_column_roles
+from framevitals.quality_diagnostics import run_quality_diagnostics
+
+
+_SAMPLE_FINDING_KEYS = (
+    "identifier_duplicates",
+    "quasi_constant_columns",
+    "coercion_candidates",
+    "category_normalisation",
+    "blank_strings",
+    "infinite_values",
+    "mixed_object_types",
+    "missingness_relationships",
+)
+
+
+def _annotate_sample_findings(
+    payload: dict[str, Any],
+    *,
+    source_rows: int,
+    sample_rows: int,
+) -> None:
+    sampled = sample_rows < source_rows
+    if not sampled:
+        return
+
+    for key in _SAMPLE_FINDING_KEYS:
+        findings = payload.get(key, [])
+        if not isinstance(findings, list):
+            continue
+        for finding in findings:
+            if not isinstance(finding, dict):
+                continue
+            finding["sampled"] = True
+            finding["sample_rows"] = sample_rows
+            finding["source_rows"] = source_rows
+            if key == "identifier_duplicates":
+                finding["count_semantics"] = "lower_bound_from_sample"
+
+    primary_keys = payload.get("primary_key_candidates", [])
+    if isinstance(primary_keys, list):
+        for finding in primary_keys:
+            if not isinstance(finding, dict):
+                continue
+            finding["confidence"] = "candidate"
+            finding["sampled"] = True
+            finding["sample_rows"] = sample_rows
+            finding["source_rows"] = source_rows
+            finding["full_source_uniqueness_confirmed"] = False
+            finding["reason"] = (
+                "Column is complete and unique within the bounded row sample; "
+                "full-source uniqueness was not confirmed."
+            )
+
+    duplicate_columns = payload.get("duplicate_columns", [])
+    if isinstance(duplicate_columns, list):
+        for finding in duplicate_columns:
+            if not isinstance(finding, dict):
+                continue
+            finding["sampled"] = True
+            finding["sample_rows"] = sample_rows
+            finding["source_rows"] = source_rows
+            finding["confirmed_with_full_equality"] = False
+            finding["candidate_only"] = True
+            finding["confirmation_scope"] = "bounded_row_sample"
+
+
+def run_streaming_quality_diagnostics(
+    sample: pd.DataFrame,
+    *,
+    profile: Mapping[str, Any],
+    source_rows: int,
+    source_columns: int,
+    max_sample_rows: int = 5_000,
+    max_columns: int = 100,
+    max_missingness_columns: int = 25,
+) -> dict[str, Any]:
+    """Run quality diagnostics without materializing a streaming source.
+
+    Full-source missingness and duplicate-row estimates come from ``profile``.
+    Value-level diagnostics operate on ``sample``. The returned execution
+    metadata describes that split so downstream consumers can distinguish
+    observed facts from sample-derived candidates.
+    """
+    if source_rows < 1:
+        raise ValueError("source_rows must be at least 1.")
+    if source_columns < 1:
+        raise ValueError("source_columns must be at least 1.")
+
+    roles = infer_column_roles(sample)
+    payload = run_quality_diagnostics(
+        sample,
+        profile=profile,
+        column_roles=roles,
+        max_sample_rows=max_sample_rows,
+        max_columns=max_columns,
+        max_missingness_columns=max_missingness_columns,
+    )
+
+    sample_rows = int(len(sample))
+    _annotate_sample_findings(
+        payload,
+        source_rows=int(source_rows),
+        sample_rows=sample_rows,
+    )
+
+    payload["rows"] = int(source_rows)
+    payload["columns"] = int(source_columns)
+    payload["columns_checked"] = min(int(source_columns), int(max_columns))
+    payload["truncated_columns"] = int(source_columns) > int(max_columns)
+    payload["duplicate_rows"] = int(profile.get("duplicate_rows", 0) or 0)
+
+    issue_groups = (
+        "identifier_duplicates",
+        "quasi_constant_columns",
+        "duplicate_columns",
+        "coercion_candidates",
+        "category_normalisation",
+        "blank_strings",
+        "infinite_values",
+        "mixed_object_types",
+        "missingness_relationships",
+    )
+    issue_count = sum(
+        len(payload.get(key, []))
+        for key in issue_groups
+        if isinstance(payload.get(key, []), list)
+    )
+    duplicate_rows = int(payload["duplicate_rows"])
+    payload["summary"] = {
+        "issue_groups": sum(bool(payload.get(key)) for key in issue_groups),
+        "issue_count": issue_count + (1 if duplicate_rows else 0),
+        "primary_key_candidate_count": len(payload.get("primary_key_candidates", [])),
+    }
+    payload["execution"] = {
+        "method": "streaming_profile_with_bounded_quality_sample",
+        "full_materialization": False,
+        "source_rows": int(source_rows),
+        "source_columns": int(source_columns),
+        "sample_rows": sample_rows,
+        "sampled": sample_rows < int(source_rows),
+        "full_source_inputs": ["missingness", "duplicate_row_estimate"],
+        "sample_inputs": [
+            "column_roles",
+            "identifier_duplicates",
+            "quasi_constants",
+            "duplicate_column_candidates",
+            "coercion_candidates",
+            "category_normalisation",
+            "blank_strings",
+            "infinite_values",
+            "mixed_object_types",
+            "missingness_relationships",
+        ],
+        "candidate_only_checks": (
+            ["primary_key_candidates", "duplicate_columns"]
+            if sample_rows < int(source_rows)
+            else []
+        ),
+    }
+    return payload

From c3175896858c4caa7669dc289ca23d01ff0719d9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:02:22 +0530
Subject: [PATCH 185/439] feat: stream bounded quality checks for Arrow sources

---
 src/framevitals/focused.py | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index 84525c5..a2ee943 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -101,11 +101,33 @@ def quality(
     max_columns: int = 100,
     max_missingness_columns: int = 25,
 ) -> dict[str, Any]:
+    source = resolve_source(data)
+    metadata = source.inspect()
+    if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
+        from framevitals.streaming_profile import build_streaming_profile
+        from framevitals.streaming_quality import run_streaming_quality_diagnostics
+
+        dataset_profile, sample = build_streaming_profile(
+            source,
+            sample_rows=max_sample_rows,
+            return_sample=True,
+        )
+        payload = run_streaming_quality_diagnostics(
+            sample,
+            profile=dataset_profile,
+            source_rows=int(metadata.rows or len(sample)),
+            source_columns=int(metadata.columns or len(sample.columns)),
+            max_sample_rows=max_sample_rows,
+            max_columns=max_columns,
+            max_missingness_columns=max_missingness_columns,
+        )
+        return _named(payload, metadata.name)
+
     from framevitals.column_roles import infer_column_roles
     from framevitals.profiler import build_profile
     from framevitals.quality_diagnostics import run_quality_diagnostics
 
-    dataframe, source_name = _load(data)
+    dataframe = source.load()
     dataset_profile = build_profile(dataframe)
     column_roles = infer_column_roles(dataframe)
     payload = run_quality_diagnostics(
@@ -116,7 +138,7 @@ def quality(
         max_columns=max_columns,
         max_missingness_columns=max_missingness_columns,
     )
-    return _named(payload, source_name)
+    return _named(payload, metadata.name)
 
 
 def statistics(

From d5322dbc2474af1b06458306382c0578dc36df19 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:02:43 +0530
Subject: [PATCH 186/439] feat: stream ML readiness for Arrow sources

---
 src/framevitals/focused.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index a2ee943..69b042b 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -83,14 +83,23 @@ def health(data: DataInput) -> dict[str, Any]:
 
 
 def ml_readiness(data: DataInput) -> dict[str, Any]:
+    source = resolve_source(data)
+    metadata = source.inspect()
+    if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
+        from framevitals.ml_readiness import calculate_ml_readiness_from_profile
+        from framevitals.streaming_profile import build_streaming_profile
+
+        dataset_profile = build_streaming_profile(source)
+        return _named(calculate_ml_readiness_from_profile(dataset_profile), metadata.name)
+
     from framevitals.ml_readiness import calculate_ml_readiness
     from framevitals.profiler import build_profile
 
-    dataframe, source_name = _load(data)
+    dataframe = source.load()
     dataset_profile = build_profile(dataframe)
     return _named(
         calculate_ml_readiness(dataframe, profile=dataset_profile),
-        source_name,
+        metadata.name,
     )
 
 

From 7591c7c529aaeb061287b1d160ed7288109b256a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:02:54 +0530
Subject: [PATCH 187/439] test: cover streaming parquet quality semantics

---
 tests/test_parquet_streaming.py | 37 +++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
index 68df628..15fa61c 100644
--- a/tests/test_parquet_streaming.py
+++ b/tests/test_parquet_streaming.py
@@ -126,6 +126,43 @@ def fail_load(self):
     assert execution["components"]["uniqueness"] == "full_stream_sample_estimate"
 
 
+def test_public_quality_streams_parquet_and_marks_sample_candidates(tmp_path, monkeypatch):
+    path = tmp_path / "quality.parquet"
+    frame = _write_parquet(path, rows=12_000)
+    frame["other_copy"] = frame["other"]
+    pq.write_table(pa.Table.from_pandas(frame, preserve_index=False), path, row_group_size=777)
+
+    def fail_load(self):
+        raise AssertionError("quality must not materialize the complete Parquet file")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.quality(path, max_sample_rows=1_000)
+
+    assert result["dataset_name"] == "quality.parquet"
+    assert result["rows"] == len(frame)
+    assert result["columns"] == len(frame.columns)
+    execution = result["execution"]
+    assert execution["full_materialization"] is False
+    assert execution["sampled"] is True
+    assert execution["sample_rows"] == 1_000
+    assert set(execution["candidate_only_checks"]) == {
+        "primary_key_candidates",
+        "duplicate_columns",
+    }
+
+    duplicate_candidates = result["duplicate_columns"]
+    assert duplicate_candidates
+    candidate = next(
+        item
+        for item in duplicate_candidates
+        if {item["canonical_column"], *item["duplicate_columns"]}
+        >= {"other", "other_copy"}
+    )
+    assert candidate["candidate_only"] is True
+    assert candidate["confirmed_with_full_equality"] is False
+    assert candidate["confirmation_scope"] == "bounded_row_sample"
+
+
 def test_plan_reads_only_bounded_parquet_sample_and_uses_true_shape(tmp_path, monkeypatch):
     path = tmp_path / "plan.parquet"
     frame = _write_parquet(path, rows=12_000)

From cdcead2c35729f43c303825abec8a80f3930f5d8 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:03:26 +0530
Subject: [PATCH 188/439] test: verify streaming parquet ML readiness

---
 tests/test_parquet_streaming.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
index 15fa61c..e41ba85 100644
--- a/tests/test_parquet_streaming.py
+++ b/tests/test_parquet_streaming.py
@@ -108,6 +108,26 @@ def fail_load(self):
     )
 
 
+def test_public_ml_readiness_streams_parquet_without_calling_load(tmp_path, monkeypatch):
+    path = tmp_path / "ml-readiness.parquet"
+    frame = _write_parquet(path, rows=12_000)
+
+    def fail_load(self):
+        raise AssertionError("ML readiness must not materialize the complete Parquet file")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.ml_readiness(path)
+
+    assert result["dataset_name"] == "ml-readiness.parquet"
+    assert result["numeric_columns"] == ["value", "other"]
+    assert result["categorical_columns"] == ["group"]
+    expected_missing = int(frame["value"].isna().sum()) / (len(frame) * 4) * 100
+    assert result["issues"]["missing_percent"] == pytest.approx(
+        round(expected_missing, 2)
+    )
+    assert 0 <= result["score"] <= 100
+
+
 def test_large_public_health_discloses_bounded_outlier_estimate(tmp_path, monkeypatch):
     path = tmp_path / "large-health.parquet"
     frame = _write_parquet(path, rows=60_000)

From 84f714ea85cc1fd1c022dc03fdf5afad704ec01a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:04:10 +0530
Subject: [PATCH 189/439] fix: report true source size for streaming
 relationships

---
 src/framevitals/focused.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index 69b042b..6f3bc50 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -237,8 +237,15 @@ def relationships(
             max_candidate_pairs=max_candidate_pairs,
             max_edges_returned=max_edges_returned,
         )
-        payload["source_rows"] = metadata.rows
-        payload["source_columns"] = metadata.columns
+        sample_metadata = payload.setdefault("sample", {})
+        sample_metadata.update({
+            "source_rows": int(metadata.rows or len(sample)),
+            "sample_rows": int(len(sample)),
+            "sampled": bool((metadata.rows or len(sample)) > len(sample)),
+            "full_materialization": False,
+            "strategy": "streaming_evenly_spaced_global_rows",
+        })
+        payload["source"] = metadata.to_dict()
         payload["streaming_source"] = True
         payload["full_materialization"] = False
         return _named(payload, metadata.name)

From b8368f60fd02d23c03b80a802ee16169ad102d7a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:04:49 +0530
Subject: [PATCH 190/439] feat: disclose streaming ML readiness provenance

---
 src/framevitals/ml_readiness.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/ml_readiness.py b/src/framevitals/ml_readiness.py
index 8b2ddb0..1d18c62 100644
--- a/src/framevitals/ml_readiness.py
+++ b/src/framevitals/ml_readiness.py
@@ -72,7 +72,7 @@ def calculate_ml_readiness_from_profile(profile: dict[str, Any]) -> dict[str, An
     categorical_cols = list(profile.get("categorical_columns", []))
     numeric_cols = list(profile.get("numeric_columns", []))
 
-    return _score_ml_readiness(
+    result = _score_ml_readiness(
         rows=rows,
         columns=columns,
         missing_total=missing_total,
@@ -81,6 +81,27 @@ def calculate_ml_readiness_from_profile(profile: dict[str, Any]) -> dict[str, An
         numeric_cols=numeric_cols,
     )
 
+    streaming = profile.get("streaming_metadata", {})
+    if streaming.get("enabled"):
+        duplicate_metadata = profile.get("duplicate_metadata", {})
+        duplicate_sampled = bool(duplicate_metadata.get("sampled"))
+        result["execution"] = {
+            "method": "streaming_profile",
+            "full_materialization": bool(streaming.get("full_materialization", False)),
+            "source_rows": rows,
+            "source_columns": columns,
+            "sample_rows": int(streaming.get("sample_rows", 0) or 0),
+            "components": {
+                "missingness": "full_stream_exact",
+                "column_groups": "schema_exact",
+                "duplicate_rate": (
+                    "bounded_row_sample_estimate" if duplicate_sampled else "exact"
+                ),
+            },
+        }
+
+    return result
+
 
 def calculate_ml_readiness(df, profile=None):
     """Calculate ML-readiness while reusing profile metrics when available.

From 5e8529c6fd3b5fd15668286a2da3a9d56bffe706 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:05:15 +0530
Subject: [PATCH 191/439] test: disclose streaming ML readiness provenance

---
 tests/test_parquet_streaming.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
index e41ba85..30dd378 100644
--- a/tests/test_parquet_streaming.py
+++ b/tests/test_parquet_streaming.py
@@ -125,9 +125,34 @@ def fail_load(self):
     assert result["issues"]["missing_percent"] == pytest.approx(
         round(expected_missing, 2)
     )
+    execution = result["execution"]
+    assert execution["method"] == "streaming_profile"
+    assert execution["full_materialization"] is False
+    assert execution["source_rows"] == len(frame)
+    assert execution["sample_rows"] == len(frame)
+    assert execution["components"]["missingness"] == "full_stream_exact"
+    assert execution["components"]["column_groups"] == "schema_exact"
+    assert execution["components"]["duplicate_rate"] == "exact"
     assert 0 <= result["score"] <= 100
 
 
+def test_large_public_ml_readiness_discloses_duplicate_estimate(tmp_path, monkeypatch):
+    path = tmp_path / "large-ml-readiness.parquet"
+    frame = _write_parquet(path, rows=60_000)
+
+    def fail_load(self):
+        raise AssertionError("large ML readiness must stay on the streaming path")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.ml_readiness(path)
+
+    execution = result["execution"]
+    assert execution["full_materialization"] is False
+    assert execution["source_rows"] == len(frame)
+    assert execution["sample_rows"] == 50_000
+    assert execution["components"]["duplicate_rate"] == "bounded_row_sample_estimate"
+
+
 def test_large_public_health_discloses_bounded_outlier_estimate(tmp_path, monkeypatch):
     path = tmp_path / "large-health.parquet"
     frame = _write_parquet(path, rows=60_000)

From 3f4d5e3d35fa152b81abe263966765041fe1a232 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:06:43 +0530
Subject: [PATCH 192/439] feat: add streaming-aware column role inference

---
 src/framevitals/streaming_roles.py | 201 +++++++++++++++++++++++++++++
 1 file changed, 201 insertions(+)
 create mode 100644 src/framevitals/streaming_roles.py

diff --git a/src/framevitals/streaming_roles.py b/src/framevitals/streaming_roles.py
new file mode 100644
index 0000000..5b1deac
--- /dev/null
+++ b/src/framevitals/streaming_roles.py
@@ -0,0 +1,201 @@
+"""Column-role inference for streaming dataset sources.
+
+Semantic/value-pattern inference remains bounded, while full-source profile facts
+(missingness and categorical cardinality when available) correct the sample
+roles. Every column records the scope of its cardinality evidence.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+import pandas as pd
+
+from framevitals.column_roles import (
+    ID_KEYWORDS,
+    _classify_missingness,
+    _name_matches,
+    infer_column_roles,
+    summarize_roles,
+)
+
+
+_CARDINALITY_ROLES = {
+    "constant",
+    "binary",
+    "low_cardinality",
+    "high_cardinality",
+    "unique_like",
+}
+_MISSINGNESS_ROLES = {
+    "complete",
+    "low_missing",
+    "moderate_missing",
+    "high_missing",
+    "very_high_missing",
+    "severe_missing",
+}
+_DERIVED_ROLES = {
+    "analysis_candidate",
+    "target_candidate",
+    "regression_target_candidate",
+}
+
+
+def _categorical_cardinality(
+    profile: Mapping[str, Any],
+    column: str,
+) -> tuple[int | None, str, bool]:
+    summaries = profile.get("categorical_summary", {})
+    if not isinstance(summaries, Mapping):
+        return None, "unavailable", True
+    raw = summaries.get(column)
+    if not isinstance(raw, Mapping) or "unique_values" not in raw:
+        return None, "unavailable", True
+
+    unique_count = int(raw.get("unique_values", 0) or 0)
+    approximate = bool(raw.get("approximate", False))
+    method = str(raw.get("unique_values_method") or "streaming_profile")
+    return unique_count, method, approximate
+
+
+def _apply_cardinality_roles(
+    roles: set[str],
+    *,
+    column: str,
+    unique_count: int,
+    rows: int,
+    is_numeric: bool,
+    semantic_type: str | None,
+) -> None:
+    unique_ratio = unique_count / max(rows, 1)
+    if unique_count <= 1:
+        roles.add("constant")
+    if unique_count == 2:
+        roles.add("binary")
+    if 2 <= unique_count <= 10:
+        roles.add("low_cardinality")
+    if unique_ratio > 0.80:
+        roles.add("high_cardinality")
+    if unique_ratio > 0.95:
+        roles.add("unique_like")
+
+    id_from_name = _name_matches(column, ID_KEYWORDS)
+    if semantic_type == "uuid":
+        roles.add("id_like")
+    elif id_from_name:
+        if not is_numeric or unique_ratio > 0.99:
+            roles.add("id_like")
+    elif unique_ratio > 0.95 and not is_numeric:
+        roles.add("id_like")
+
+
+def _recompute_derived_roles(
+    roles: set[str],
+    *,
+    is_numeric: bool,
+    unique_count: int,
+) -> None:
+    roles.difference_update(_DERIVED_ROLES)
+    excluded = {"id_like", "time_like", "sequence_like", "constant"}
+    if not roles.intersection(excluded):
+        roles.add("analysis_candidate")
+
+    if "id_like" not in roles and "constant" not in roles:
+        if "binary" in roles or "low_cardinality" in roles:
+            roles.add("target_candidate")
+        elif is_numeric and unique_count > 10:
+            roles.add("regression_target_candidate")
+
+
+def infer_streaming_column_roles(
+    sample: pd.DataFrame,
+    *,
+    profile: Mapping[str, Any],
+) -> dict[str, Any]:
+    """Infer roles from a bounded sample corrected by full-stream profile facts."""
+    source_rows = int(profile.get("shape", {}).get("rows", len(sample)) or len(sample))
+    sample_rows = int(len(sample))
+    sampled = sample_rows < source_rows
+    missing_counts = profile.get("missing_counts", {})
+    if not isinstance(missing_counts, Mapping):
+        missing_counts = {}
+
+    sample_roles = infer_column_roles(sample)
+    columns: dict[str, Any] = {}
+
+    for column, raw_info in sample_roles.items():
+        info = dict(raw_info)
+        roles = set(info.get("roles", []))
+        roles.difference_update(_MISSINGNESS_ROLES)
+        roles.difference_update(_CARDINALITY_ROLES)
+
+        missing_count = int(missing_counts.get(column, sample[column].isna().sum()) or 0)
+        non_missing = max(source_rows - missing_count, 0)
+        missing_percent = round(missing_count / max(source_rows, 1) * 100, 2)
+        roles.add(_classify_missingness(missing_percent))
+
+        categorical_unique, categorical_method, categorical_approximate = (
+            _categorical_cardinality(profile, column)
+        )
+        if categorical_unique is not None:
+            unique_count = categorical_unique
+            cardinality_scope = (
+                "full_stream_approximate" if categorical_approximate else "full_stream_exact"
+            )
+            cardinality_method = categorical_method
+            cardinality_approximate = categorical_approximate
+        else:
+            unique_count = int(sample[column].nunique(dropna=True))
+            cardinality_scope = "bounded_row_sample" if sampled else "full_source"
+            cardinality_method = (
+                "evenly_spaced_row_sample" if sampled else "exact_full_source_sample"
+            )
+            cardinality_approximate = sampled
+
+        # Sample uniqueness may have introduced id_like. Remove that role unless
+        # it is justified again by source-corrected cardinality/name/semantics.
+        roles.discard("id_like")
+        _apply_cardinality_roles(
+            roles,
+            column=str(column),
+            unique_count=unique_count,
+            rows=source_rows,
+            is_numeric=bool(info.get("is_numeric")),
+            semantic_type=info.get("semantic_type"),
+        )
+        _recompute_derived_roles(
+            roles,
+            is_numeric=bool(info.get("is_numeric")),
+            unique_count=unique_count,
+        )
+
+        info.update({
+            "roles": sorted(roles),
+            "missing_percent": missing_percent,
+            "non_missing_count": non_missing,
+            "unique_count": unique_count,
+            "unique_ratio": round(unique_count / max(source_rows, 1), 4),
+            "cardinality_scope": cardinality_scope,
+            "cardinality_method": cardinality_method,
+            "cardinality_approximate": cardinality_approximate,
+            "semantic_scope": "bounded_row_sample" if sampled else "full_source",
+            "sample_rows": sample_rows,
+            "source_rows": source_rows,
+        })
+        columns[str(column)] = info
+
+    return {
+        "columns": columns,
+        "summary": summarize_roles(columns),
+        "execution": {
+            "method": "streaming_profile_with_bounded_semantic_sample",
+            "full_materialization": False,
+            "source_rows": source_rows,
+            "sample_rows": sample_rows,
+            "sampled": sampled,
+            "full_source_inputs": ["missingness", "categorical_cardinality"],
+            "sample_inputs": ["semantic_patterns", "numeric_cardinality_when_needed"],
+        },
+    }

From 340649cb829dade6513248b1468bcc3e7cbf6c1d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:07:09 +0530
Subject: [PATCH 193/439] feat: stream column role inference for Arrow sources

---
 src/framevitals/focused.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index 6f3bc50..581b009 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -52,12 +52,26 @@ def profile(data: DataInput) -> dict[str, Any]:
 
 
 def roles(data: DataInput) -> dict[str, Any]:
+    source = resolve_source(data)
+    metadata = source.inspect()
+    if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
+        from framevitals.streaming_profile import build_streaming_profile
+        from framevitals.streaming_roles import infer_streaming_column_roles
+
+        dataset_profile, sample = build_streaming_profile(
+            source,
+            sample_rows=5_000,
+            return_sample=True,
+        )
+        payload = infer_streaming_column_roles(sample, profile=dataset_profile)
+        return _named(payload, metadata.name)
+
     from framevitals.column_roles import infer_column_roles, summarize_roles
 
-    dataframe, source_name = _load(data)
+    dataframe = source.load()
     column_roles = infer_column_roles(dataframe)
     return {
-        "dataset_name": source_name,
+        "dataset_name": metadata.name,
         "columns": column_roles,
         "summary": summarize_roles(column_roles),
     }

From ff7ca3a71dc33d1c86e5856765460cf7cdd2bcdb Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:07:46 +0530
Subject: [PATCH 194/439] test: verify streaming parquet role inference

---
 tests/test_parquet_streaming.py | 34 +++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
index 30dd378..6f8fe45 100644
--- a/tests/test_parquet_streaming.py
+++ b/tests/test_parquet_streaming.py
@@ -68,6 +68,40 @@ def fail_load(self):
     assert "event_time" in result["date_columns"]
 
 
+def test_public_roles_stream_parquet_without_calling_load(tmp_path, monkeypatch):
+    path = tmp_path / "roles.parquet"
+    frame = _write_parquet(path, rows=12_000)
+
+    def fail_load(self):
+        raise AssertionError("roles must not materialize the complete Parquet file")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.roles(path)
+
+    assert result["dataset_name"] == "roles.parquet"
+    execution = result["execution"]
+    assert execution["full_materialization"] is False
+    assert execution["source_rows"] == len(frame)
+    assert execution["sample_rows"] == 5_000
+    assert execution["sampled"] is True
+
+    value = result["columns"]["value"]
+    expected_missing = int(frame["value"].isna().sum()) / len(frame) * 100
+    assert value["missing_percent"] == pytest.approx(round(expected_missing, 2))
+    assert value["source_rows"] == len(frame)
+    assert value["sample_rows"] == 5_000
+    assert value["cardinality_scope"] == "bounded_row_sample"
+    assert value["cardinality_approximate"] is True
+
+    group = result["columns"]["group"]
+    assert group["unique_count"] == 7
+    assert "low_cardinality" in group["roles"]
+    assert group["cardinality_scope"] in {
+        "bounded_row_sample",
+        "full_stream_approximate",
+    }
+
+
 def test_large_parquet_retains_only_bounded_row_sample(tmp_path):
     path = tmp_path / "large.parquet"
     frame = _write_parquet(path, rows=60_000)

From 4ecf320aea200a53126485bb09fa6a5237f81f68 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:08:17 +0530
Subject: [PATCH 195/439] fix: distinguish sampled and native role cardinality

---
 src/framevitals/streaming_roles.py | 56 ++++++++++++++++++++++--------
 1 file changed, 41 insertions(+), 15 deletions(-)

diff --git a/src/framevitals/streaming_roles.py b/src/framevitals/streaming_roles.py
index 5b1deac..8f8635d 100644
--- a/src/framevitals/streaming_roles.py
+++ b/src/framevitals/streaming_roles.py
@@ -1,8 +1,9 @@
 """Column-role inference for streaming dataset sources.
 
 Semantic/value-pattern inference remains bounded, while full-source profile facts
-(missingness and categorical cardinality when available) correct the sample
-roles. Every column records the scope of its cardinality evidence.
+correct the sample roles where the streaming profiler has authoritative or
+full-stream sketch evidence. Every column records the scope of its cardinality
+evidence.
 """
 
 from __future__ import annotations
@@ -46,18 +47,38 @@
 def _categorical_cardinality(
     profile: Mapping[str, Any],
     column: str,
-) -> tuple[int | None, str, bool]:
+    *,
+    sampled: bool,
+) -> tuple[int | None, str, str, bool]:
     summaries = profile.get("categorical_summary", {})
     if not isinstance(summaries, Mapping):
-        return None, "unavailable", True
+        return None, "unavailable", "unavailable", True
     raw = summaries.get(column)
     if not isinstance(raw, Mapping) or "unique_values" not in raw:
-        return None, "unavailable", True
+        return None, "unavailable", "unavailable", True
+
+    metadata = profile.get("categorical_summary_metadata", {})
+    if not isinstance(metadata, Mapping):
+        metadata = {}
+    native_columns = set(metadata.get("native_full_stream_columns", []) or [])
+    fallback_columns = set(metadata.get("sample_fallback_columns", []) or [])
 
     unique_count = int(raw.get("unique_values", 0) or 0)
     approximate = bool(raw.get("approximate", False))
-    method = str(raw.get("unique_values_method") or "streaming_profile")
-    return unique_count, method, approximate
+    method = str(raw.get("unique_values_method") or metadata.get("method") or "streaming_profile")
+
+    if column in native_columns:
+        return unique_count, "full_stream_approximate", method, True
+    if column in fallback_columns:
+        scope = "bounded_row_sample" if sampled else "full_source"
+        fallback_method = "evenly_spaced_row_sample" if sampled else "exact_full_source_sample"
+        return unique_count, scope, fallback_method, sampled
+
+    # A categorical result without explicit fallback metadata is exact only when
+    # the retained sample covers the entire source. Otherwise keep it scoped as
+    # approximate instead of silently promoting a sample statistic.
+    scope = "bounded_row_sample" if sampled else "full_source"
+    return unique_count, scope, method, approximate or sampled
 
 
 def _apply_cardinality_roles(
@@ -136,14 +157,15 @@ def infer_streaming_column_roles(
         missing_percent = round(missing_count / max(source_rows, 1) * 100, 2)
         roles.add(_classify_missingness(missing_percent))
 
-        categorical_unique, categorical_method, categorical_approximate = (
-            _categorical_cardinality(profile, column)
-        )
+        (
+            categorical_unique,
+            categorical_scope,
+            categorical_method,
+            categorical_approximate,
+        ) = _categorical_cardinality(profile, column, sampled=sampled)
         if categorical_unique is not None:
             unique_count = categorical_unique
-            cardinality_scope = (
-                "full_stream_approximate" if categorical_approximate else "full_stream_exact"
-            )
+            cardinality_scope = categorical_scope
             cardinality_method = categorical_method
             cardinality_approximate = categorical_approximate
         else:
@@ -195,7 +217,11 @@ def infer_streaming_column_roles(
             "source_rows": source_rows,
             "sample_rows": sample_rows,
             "sampled": sampled,
-            "full_source_inputs": ["missingness", "categorical_cardinality"],
-            "sample_inputs": ["semantic_patterns", "numeric_cardinality_when_needed"],
+            "full_source_inputs": ["missingness", "native_categorical_sketches"],
+            "sample_inputs": [
+                "semantic_patterns",
+                "numeric_cardinality",
+                "categorical_cardinality_without_native_sketch",
+            ],
         },
     }

From 49a59d207e079bcdbefd20636b27afdec9434538 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:10:32 +0530
Subject: [PATCH 196/439] feat: stream bounded statistics for Arrow sources

---
 src/framevitals/focused.py | 51 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 49 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index 581b009..c8583f1 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -174,7 +174,54 @@ def statistics(
     from framevitals.budgeted_analysis import run_budgeted_deep_statistics
     from framevitals.execution import derive_execution_budget
 
-    dataframe, source_name = _load(data)
+    source = resolve_source(data)
+    metadata = source.inspect()
+    if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
+        from framevitals.streaming_profile import sample_streaming_source
+
+        if metadata.rows is None or metadata.columns is None:
+            raise ValueError("Streaming statistics require source shape metadata.")
+
+        source_rows = int(metadata.rows)
+        source_columns = int(metadata.columns)
+        budget = derive_execution_budget(source_rows, source_columns, mode=mode)
+        sample_limit = max(
+            20,
+            min(
+                budget.deep_statistics_sample_rows,
+                budget.bootstrap_sample_rows,
+            ),
+        )
+        sample = sample_streaming_source(source, sample_rows=sample_limit)
+        payload = run_budgeted_deep_statistics(
+            sample,
+            budget=budget,
+            max_pairs=max_pairs,
+        )
+        execution = payload.setdefault("execution", {})
+        sampled = len(sample) < source_rows
+        execution.update({
+            "sampled": sampled,
+            "source_rows": source_rows,
+            "source_columns": source_columns,
+            "sample_rows": int(len(sample)),
+            "strategy": (
+                "streaming_evenly_spaced_global_rows"
+                if sampled
+                else "full_stream_via_batches"
+            ),
+            "full_materialization": False,
+            "reason": (
+                "Deep statistics ran on a bounded evenly spaced sample selected "
+                "directly from the streaming source."
+                if sampled
+                else "The streaming source fits within the deep-statistics execution budget."
+            ),
+        })
+        payload["source"] = metadata.to_dict()
+        return _named(payload, metadata.name)
+
+    dataframe = source.load()
     budget = derive_execution_budget(
         len(dataframe),
         len(dataframe.columns),
@@ -185,7 +232,7 @@ def statistics(
         budget=budget,
         max_pairs=max_pairs,
     )
-    return _named(payload, source_name)
+    return _named(payload, metadata.name)
 
 
 def anomalies(

From 3a9676624781dcc72554891695316f82086c834d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:10:48 +0530
Subject: [PATCH 197/439] test: verify bounded streaming statistics

---
 tests/test_statistics_streaming.py | 47 ++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 tests/test_statistics_streaming.py

diff --git a/tests/test_statistics_streaming.py b/tests/test_statistics_streaming.py
new file mode 100644
index 0000000..88c1a9b
--- /dev/null
+++ b/tests/test_statistics_streaming.py
@@ -0,0 +1,47 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+pa = pytest.importorskip("pyarrow")
+pq = pytest.importorskip("pyarrow.parquet")
+
+import framevitals
+from framevitals.sources import ParquetSource
+
+
+def _write_statistics_parquet(path, rows: int = 12_000) -> pd.DataFrame:
+    frame = pd.DataFrame({
+        "value": np.linspace(1.0, 100.0, rows),
+        "other": np.linspace(3.0, 203.0, rows),
+        "group": [f"g-{index % 5}" for index in range(rows)],
+        "event_time": pd.date_range("2026-01-01", periods=rows, freq="min"),
+    })
+    pq.write_table(
+        pa.Table.from_pandas(frame, preserve_index=False),
+        path,
+        row_group_size=777,
+    )
+    return frame
+
+
+def test_public_statistics_streams_bounded_parquet_sample(tmp_path, monkeypatch):
+    path = tmp_path / "statistics.parquet"
+    frame = _write_statistics_parquet(path)
+
+    def fail_load(self):
+        raise AssertionError("statistics must not materialize the complete Parquet file")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.statistics(path, mode="quick", max_pairs=2)
+
+    assert result["dataset_name"] == "statistics.parquet"
+    execution = result["execution"]
+    assert execution["scope"] == "bounded_deep_statistics"
+    assert execution["full_materialization"] is False
+    assert execution["source_rows"] == len(frame)
+    assert execution["source_columns"] == len(frame.columns)
+    assert execution["sample_rows"] == 1_000
+    assert execution["sampled"] is True
+    assert execution["strategy"] == "streaming_evenly_spaced_global_rows"
+    assert execution["pair_budget"] == 2
+    assert result["source"]["supports_streaming"] is True

From f8d94eb38000c7479d09e8ac64eba87c5b204d5c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:11:14 +0530
Subject: [PATCH 198/439] feat: add bounded full-analysis path for streaming
 sources

---
 src/framevitals/streaming_pipeline.py | 319 ++++++++++++++++++++++++++
 1 file changed, 319 insertions(+)
 create mode 100644 src/framevitals/streaming_pipeline.py

diff --git a/src/framevitals/streaming_pipeline.py b/src/framevitals/streaming_pipeline.py
new file mode 100644
index 0000000..1a76fff
--- /dev/null
+++ b/src/framevitals/streaming_pipeline.py
@@ -0,0 +1,319 @@
+"""Full FrameVitals orchestration for streaming dataset sources.
+
+The streaming pipeline performs one full-source Arrow scan to build reusable
+profile state, retains one bounded deterministic working sample, and runs legacy
+row-dependent modules only on that sample. Full-source facts are overlaid back
+onto the stable analysis result shape, and every bounded module is explicitly
+scoped so downstream consumers never mistake sample-derived diagnostics for
+full-data execution.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Iterable
+from typing import Any
+
+from framevitals.analysis_selector import select_analyses
+from framevitals.column_roles import summarize_roles
+from framevitals.config import VALID_MODULES
+from framevitals.dataset_signals import detect_dataset_signals
+from framevitals.execution import derive_execution_budget
+from framevitals.health_score import calculate_health_score_from_profile_sample
+from framevitals.ml_readiness import calculate_ml_readiness_from_profile
+from framevitals.pipeline import run_full_analysis
+from framevitals.signal_engine import build_signals
+from framevitals.sources import StreamingDatasetSource
+from framevitals.streaming_profile import build_streaming_profile
+from framevitals.streaming_quality import run_streaming_quality_diagnostics
+from framevitals.streaming_roles import infer_streaming_column_roles
+
+
+_BOUNDED_RESULT_KEYS = (
+    "advanced",
+    "deep_statistics_v2",
+    "anomalies_v2",
+    "target_intelligence",
+    "model_leaderboard",
+    "explainability",
+    "time_series",
+    "text_profile",
+)
+
+
+def _working_sample_rows(budget) -> int:
+    """Choose one reusable row sample large enough for every bounded module."""
+    requested = max(
+        budget.quality_sample_rows,
+        budget.deep_statistics_sample_rows,
+        budget.anomaly_sample_rows,
+        budget.time_series_sample_rows,
+        budget.pair_sample_rows,
+        1,
+    )
+    return min(int(budget.rows), int(requested))
+
+
+def _scope_bounded_result(
+    value: Any,
+    *,
+    source_rows: int,
+    sample_rows: int,
+    strategy: str,
+) -> Any:
+    if not isinstance(value, dict):
+        return value
+    scoped = dict(value)
+    scoped["execution_scope"] = {
+        "scope": "bounded_row_sample" if sample_rows < source_rows else "full_source",
+        "full_materialization": False,
+        "source_rows": int(source_rows),
+        "sample_rows": int(sample_rows),
+        "sampled": bool(sample_rows < source_rows),
+        "strategy": strategy,
+    }
+    return scoped
+
+
+def _streaming_cleaning_result(
+    *,
+    enabled_by_user: bool,
+    profile: dict[str, Any],
+    health: dict[str, Any],
+) -> dict[str, Any]:
+    missing_count = sum(
+        int(value)
+        for value in profile.get("missing_counts", {}).values()
+        if value is not None
+    )
+    duplicate_count = int(profile.get("duplicate_rows", 0) or 0)
+    if enabled_by_user:
+        reason = (
+            "Full-source cleaning is not executed implicitly on the streaming analysis "
+            "path because applying sample-derived mutations would be unsafe. Run the "
+            "explicit cleaning workflow when a transformed dataset is required."
+        )
+        status = "deferred_streaming"
+    else:
+        reason = "Disabled by configuration."
+        status = "disabled"
+    return {
+        "available": False,
+        "skipped": True,
+        "module": "cleaning",
+        "reason": reason,
+        "streaming_status": status,
+        "actions": [],
+        "before_health": health,
+        "after_health": health,
+        "output_path": None,
+        "missing_before": missing_count,
+        "missing_after": missing_count,
+        "duplicates_before": duplicate_count,
+        "duplicates_after": duplicate_count,
+    }
+
+
+def run_streaming_analysis(
+    *,
+    source: StreamingDatasetSource,
+    dataset_id: str,
+    original_filename: str,
+    analysis_mode: str,
+    target_column: str | None,
+    parallel_workers: int,
+    skip_ai: bool,
+    disabled_modules: Iterable[str] | None = None,
+) -> dict[str, Any]:
+    """Run a stable-shape full analysis without materializing the complete source."""
+    overall_start = time.perf_counter()
+    metadata = source.inspect()
+    rows = int(metadata.rows or 0)
+    columns = int(metadata.columns or 0)
+    if rows < 1 or columns < 1:
+        raise ValueError(f"Dataset is empty or has no columns: {metadata.name}")
+
+    user_disabled = set(disabled_modules or ())
+    unknown = sorted(user_disabled - VALID_MODULES)
+    if unknown:
+        raise ValueError("Unknown disabled module(s): " + ", ".join(unknown))
+
+    budget = derive_execution_budget(rows, columns, mode=analysis_mode)
+    working_rows = _working_sample_rows(budget)
+
+    profile_start = time.perf_counter()
+    profile, working_sample = build_streaming_profile(
+        source,
+        sample_rows=working_rows,
+        return_sample=True,
+    )
+    streaming_profile_ms = (time.perf_counter() - profile_start) * 1000
+
+    if working_sample.empty:
+        raise ValueError(f"Streaming source produced no usable rows: {metadata.name}")
+    if target_column is not None and target_column not in working_sample.columns:
+        raise ValueError(f"Target column not found: {target_column}")
+
+    role_payload = infer_streaming_column_roles(working_sample, profile=profile)
+    column_roles = role_payload["columns"]
+    roles_summary = summarize_roles(column_roles)
+    health = calculate_health_score_from_profile_sample(profile, working_sample)
+    ml_readiness = calculate_ml_readiness_from_profile(profile)
+    quality_diagnostics = (
+        run_streaming_quality_diagnostics(
+            working_sample,
+            profile=profile,
+            source_rows=rows,
+            source_columns=columns,
+            max_sample_rows=max(budget.quality_sample_rows, 10),
+        )
+        if "quality_diagnostics" not in user_disabled
+        else {
+            "available": False,
+            "skipped": True,
+            "module": "quality_diagnostics",
+            "reason": "Disabled by configuration.",
+        }
+    )
+
+    dataset_signals = detect_dataset_signals(
+        working_sample,
+        profile,
+        column_roles=column_roles,
+        source_shape=(rows, columns),
+    )
+    analysis_selection = select_analyses(
+        signals=dataset_signals,
+        analysis_mode=analysis_mode,
+        target_column=target_column,
+    )
+    analysis_selection["execution_modules"] = {
+        "disabled": sorted(user_disabled),
+        "enabled": sorted(VALID_MODULES - user_disabled),
+    }
+    analysis_selection["execution_budget"] = budget.to_dict()
+    analysis_selection["streaming"] = {
+        "enabled": True,
+        "full_materialization": False,
+        "working_sample_rows": int(len(working_sample)),
+        "source_rows": rows,
+    }
+
+    # Cleaning/charts are internally suppressed on the bounded sample. Charts
+    # are irrelevant here because this path is only selected when artifacts are
+    # disabled; cleaning would otherwise describe a transformed sample as though
+    # it represented the complete source.
+    internal_disabled = set(user_disabled) | {"cleaning", "charts"}
+    sample_payload = run_full_analysis(
+        dataset_id=dataset_id,
+        original_filename=original_filename,
+        analysis_mode=analysis_mode,
+        skip_ai=skip_ai,
+        target_column=target_column,
+        parallel_workers=parallel_workers,
+        dataframe=working_sample,
+        write_artifacts=False,
+        disabled_modules=tuple(sorted(internal_disabled)),
+    )
+
+    sample_rows = int(len(working_sample))
+    sample_strategy = "streaming_evenly_spaced_global_rows"
+    for key in _BOUNDED_RESULT_KEYS:
+        sample_payload[key] = _scope_bounded_result(
+            sample_payload.get(key),
+            source_rows=rows,
+            sample_rows=sample_rows,
+            strategy=sample_strategy,
+        )
+
+    advanced = sample_payload.get("advanced")
+    signals = build_signals(profile, health, ml_readiness, advanced)
+
+    cleaning = _streaming_cleaning_result(
+        enabled_by_user="cleaning" not in user_disabled,
+        profile=profile,
+        health=health,
+    )
+
+    execution = dict(sample_payload.get("execution", {}))
+    module_status = dict(execution.get("module_status", {}))
+    module_status["cleaning"] = (
+        "disabled" if "cleaning" in user_disabled else "deferred_streaming"
+    )
+    module_status["charts"] = (
+        "disabled" if "charts" in user_disabled else "not_applicable"
+    )
+    if "quality_diagnostics" in user_disabled:
+        module_status["quality_diagnostics"] = "disabled"
+    else:
+        module_status["quality_diagnostics"] = "ran"
+
+    module_scope: dict[str, str] = {}
+    for module in (
+        "deep_statistics",
+        "anomaly_detection",
+        "time_series",
+        "text_profile",
+        "target_intelligence",
+        "modeling",
+        "explainability",
+    ):
+        status = module_status.get(module)
+        if status in {"ran", "scheduled", "error"}:
+            module_scope[module] = (
+                "bounded_row_sample" if sample_rows < rows else "full_source"
+            )
+    module_scope.update({
+        "profile": "full_stream",
+        "column_roles": "full_stream_plus_bounded_semantic_sample",
+        "health": "full_stream_plus_bounded_outlier_sample",
+        "ml_readiness": "full_stream_profile",
+        "quality_diagnostics": (
+            "disabled"
+            if "quality_diagnostics" in user_disabled
+            else "full_stream_plus_bounded_value_sample"
+        ),
+        "cleaning": "deferred_streaming" if "cleaning" not in user_disabled else "disabled",
+        "charts": "not_applicable" if "charts" not in user_disabled else "disabled",
+    })
+
+    execution.update({
+        "disabled_modules": sorted(user_disabled),
+        "internally_disabled_modules": sorted(internal_disabled - user_disabled),
+        "module_status": module_status,
+        "module_scope": module_scope,
+        "budget": budget.to_dict(),
+        "streaming": {
+            "enabled": True,
+            "source": metadata.to_dict(),
+            "full_materialization": False,
+            "source_rows": rows,
+            "source_columns": columns,
+            "working_sample_rows": sample_rows,
+            "working_sample_strategy": sample_strategy,
+            "single_full_source_profile_scan": True,
+        },
+    })
+
+    timings = dict(sample_payload.get("timings_ms", {}))
+    timings["streaming_profile"] = streaming_profile_ms
+    timings["total"] = (time.perf_counter() - overall_start) * 1000
+
+    sample_payload.update({
+        "filename": original_filename,
+        "artifacts_enabled": False,
+        "execution": execution,
+        "profile": profile,
+        "column_roles": column_roles,
+        "roles_summary": roles_summary,
+        "dataset_signals": dataset_signals,
+        "analysis_selection": analysis_selection,
+        "health": health,
+        "signals": signals,
+        "ml_readiness": ml_readiness,
+        "quality_diagnostics": quality_diagnostics,
+        "cleaning": cleaning,
+        "charts": [],
+        "timings_ms": timings,
+    })
+    return sample_payload

From a8d7c41a8b6650539c807bdb900ea4c2ae18a6ae Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:11:23 +0530
Subject: [PATCH 199/439] ci: run streaming statistics regression

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8d0ff60..4e921e6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -230,7 +230,7 @@ jobs:
       - name: Run streaming Parquet tests without native extension
         env:
           FRAMEVITALS_BACKEND: numpy
-        run: pytest -v tests/test_parquet_streaming.py
+        run: pytest -v tests/test_parquet_streaming.py tests/test_statistics_streaming.py
 
   optional-features:
     name: optional features (Python 3.12)

From 6d1bfa47063f5f8ddbb5aba28d77222f3326d8dd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:12:32 +0530
Subject: [PATCH 200/439] feat: dispatch public analysis through streaming
 sources

---
 src/framevitals/analysis_api.py | 109 ++++++++++++++++++++++++++++++++
 1 file changed, 109 insertions(+)
 create mode 100644 src/framevitals/analysis_api.py

diff --git a/src/framevitals/analysis_api.py b/src/framevitals/analysis_api.py
new file mode 100644
index 0000000..0e05057
--- /dev/null
+++ b/src/framevitals/analysis_api.py
@@ -0,0 +1,109 @@
+"""Public full-analysis dispatcher.
+
+This module keeps input/source routing separate from the legacy analysis API.
+Streaming Arrow-capable sources use the bounded streaming orchestrator when
+artifacts are disabled; DataFrames, non-streaming files, and explicit artifact
+runs retain the existing materialized pipeline behavior.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+import pandas as pd
+
+from framevitals.config import ConfigInput, resolve_config
+from framevitals.pipeline import run_full_analysis
+from framevitals.result import AnalysisResult
+from framevitals.sources import StreamingDatasetSource, resolve_source
+
+
+DataInput = str | Path | pd.DataFrame
+
+
+def analyze(
+    data: DataInput,
+    *,
+    target: str | None = None,
+    mode: str | None = None,
+    artifacts: bool | None = None,
+    workers: int | None = None,
+    preset: str | None = None,
+    config: ConfigInput = None,
+    disabled_modules: list[str] | tuple[str, ...] | None = None,
+) -> AnalysisResult:
+    """Analyze a tabular dataset through the appropriate execution source."""
+    resolved = resolve_config(
+        config,
+        preset=preset,
+        mode=mode,
+        target=target,
+        artifacts=artifacts,
+        workers=workers,
+        disabled_modules=disabled_modules,
+    )
+    dataset_id = f"fv_{uuid4().hex[:12]}"
+
+    if isinstance(data, pd.DataFrame):
+        if data.empty:
+            raise ValueError("Dataset DataFrame is empty.")
+        payload = run_full_analysis(
+            dataset_id=dataset_id,
+            original_filename="<dataframe>",
+            analysis_mode=resolved.mode,
+            target_column=resolved.target,
+            parallel_workers=resolved.workers,
+            skip_ai=True,
+            dataframe=data,
+            write_artifacts=resolved.artifacts,
+            disabled_modules=resolved.disabled_modules,
+        )
+    elif isinstance(data, (str, Path)):
+        source = resolve_source(data)
+        metadata = source.inspect()
+        if metadata.rows == 0:
+            raise ValueError(f"Dataset is empty: {metadata.name}")
+
+        if (
+            metadata.supports_streaming
+            and isinstance(source, StreamingDatasetSource)
+            and not resolved.artifacts
+        ):
+            from framevitals.streaming_pipeline import run_streaming_analysis
+
+            payload = run_streaming_analysis(
+                source=source,
+                dataset_id=dataset_id,
+                original_filename=metadata.name,
+                analysis_mode=resolved.mode,
+                target_column=resolved.target,
+                parallel_workers=resolved.workers,
+                skip_ai=True,
+                disabled_modules=resolved.disabled_modules,
+            )
+        else:
+            path = Path(data)
+            if not path.exists():
+                raise FileNotFoundError(f"Dataset not found: {path}")
+            if not path.is_file():
+                raise ValueError(f"Expected a file for dataset, got: {path}")
+            payload = run_full_analysis(
+                dataset_id=dataset_id,
+                file_path=path,
+                original_filename=path.name,
+                analysis_mode=resolved.mode,
+                target_column=resolved.target,
+                parallel_workers=resolved.workers,
+                skip_ai=True,
+                write_artifacts=resolved.artifacts,
+                disabled_modules=resolved.disabled_modules,
+            )
+    else:
+        raise TypeError(
+            "data must be a pandas DataFrame or a path to a supported dataset."
+        )
+
+    payload["config"] = resolved.to_dict()
+    return AnalysisResult(payload)

From b1785a2c3a06bbc9c50c02a5446b2399de70d0af Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:13:46 +0530
Subject: [PATCH 201/439] feat: route public analyze through source dispatcher

---
 src/framevitals/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 8cd90fc..c2182b0 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -158,7 +158,7 @@ def analyze(
     disabled_modules: list[str] | tuple[str, ...] | None = None,
 ) -> AnalysisResult:
     """Analyze a tabular dataset from a DataFrame or supported file path."""
-    from framevitals.api import analyze as _analyze
+    from framevitals.analysis_api import analyze as _analyze
 
     return _analyze(
         data,

From 43cc64578dfd5daaa5f355a68174999341c12e93 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:14:31 +0530
Subject: [PATCH 202/439] test: verify full analyze streams parquet input

---
 tests/test_parquet_streaming.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
index 6f8fe45..04676e1 100644
--- a/tests/test_parquet_streaming.py
+++ b/tests/test_parquet_streaming.py
@@ -68,6 +68,30 @@ def fail_load(self):
     assert "event_time" in result["date_columns"]
 
 
+def test_public_analyze_streams_parquet_without_calling_load(tmp_path, monkeypatch):
+    path = tmp_path / "analyze.parquet"
+    frame = _write_parquet(path, rows=12_000)
+
+    def fail_load(self):
+        raise AssertionError("analyze must not materialize the complete Parquet file")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.analyze(path, mode="quick", artifacts=False, workers=1)
+
+    assert result["filename"] == "analyze.parquet"
+    assert result["profile"]["shape"] == {"rows": len(frame), "columns": 4}
+    assert result["dataset_signals"]["row_count"] == len(frame)
+    assert result["execution"]["streaming"]["enabled"] is True
+    assert result["execution"]["streaming"]["full_materialization"] is False
+    assert result["execution"]["streaming"]["source_rows"] == len(frame)
+    assert result["execution"]["streaming"]["working_sample_rows"] == 5_000
+    assert result["execution"]["streaming"]["single_full_source_profile_scan"] is True
+    assert result["execution"]["module_scope"]["profile"] == "full_stream"
+    assert result["cleaning"]["streaming_status"] == "deferred_streaming"
+    assert result["charts"] == []
+    assert result["config"]["artifacts"] is False
+
+
 def test_public_roles_stream_parquet_without_calling_load(tmp_path, monkeypatch):
     path = tmp_path / "roles.parquet"
     frame = _write_parquet(path, rows=12_000)

From f6eddd2adbf2b98dc6482c0a2a5518f1546aac22 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:16:26 +0530
Subject: [PATCH 203/439] feat: stream bounded anomaly analysis for Arrow
 sources

---
 src/framevitals/focused.py | 71 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 69 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index c8583f1..03952aa 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -248,7 +248,74 @@ def anomalies(
     from framevitals.budgeted_analysis import run_budgeted_anomalies
     from framevitals.execution import derive_execution_budget
 
-    dataframe, source_name = _load(data)
+    source = resolve_source(data)
+    metadata = source.inspect()
+    if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
+        from framevitals.streaming_profile import (
+            numeric_columns_for_streaming_source,
+            sample_streaming_source,
+        )
+
+        if metadata.rows is None or metadata.columns is None:
+            raise ValueError("Streaming anomaly analysis requires source shape metadata.")
+        source_rows = int(metadata.rows)
+        source_columns = int(metadata.columns)
+        budget = derive_execution_budget(source_rows, source_columns, mode=mode)
+        numeric_columns = numeric_columns_for_streaming_source(source)
+        if not numeric_columns:
+            return _named({
+                "available": False,
+                "reason": "No numeric columns available for anomaly analysis.",
+                "execution": {
+                    "sampled": False,
+                    "source_rows": source_rows,
+                    "source_columns": source_columns,
+                    "sample_rows": 0,
+                    "full_materialization": False,
+                    "strategy": "streaming_schema_only",
+                },
+                "source": metadata.to_dict(),
+            }, metadata.name)
+
+        sample_limit = max(100, int(budget.anomaly_sample_rows))
+        sample = sample_streaming_source(
+            source,
+            sample_rows=sample_limit,
+            columns=numeric_columns,
+        )
+        payload = run_budgeted_anomalies(
+            sample,
+            budget=budget,
+            contamination=contamination,
+            threshold=threshold,
+            max_columns=max_columns,
+            top_k=top_k,
+        )
+        execution = payload.setdefault("execution", {})
+        sampled = len(sample) < source_rows
+        execution.update({
+            "sampled": sampled,
+            "source_rows": source_rows,
+            "source_columns": source_columns,
+            "projected_columns": int(len(numeric_columns)),
+            "sample_rows": int(len(sample)),
+            "strategy": (
+                "streaming_evenly_spaced_numeric_projection"
+                if sampled
+                else "full_stream_numeric_projection"
+            ),
+            "full_materialization": False,
+            "reason": (
+                "Anomaly diagnostics ran on a bounded numeric projection selected "
+                "directly from the streaming source."
+                if sampled
+                else "The streaming source fits within the anomaly execution budget."
+            ),
+        })
+        payload["source"] = metadata.to_dict()
+        return _named(payload, metadata.name)
+
+    dataframe = source.load()
     budget = derive_execution_budget(
         len(dataframe),
         len(dataframe.columns),
@@ -262,7 +329,7 @@ def anomalies(
         max_columns=max_columns,
         top_k=top_k,
     )
-    return _named(payload, source_name)
+    return _named(payload, metadata.name)
 
 
 def relationships(

From 6d7d1529f0383fd8e60fb2729aab927bdff2cb82 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:17:07 +0530
Subject: [PATCH 204/439] test: verify anomaly analysis streams numeric parquet
 projection

---
 tests/test_parquet_streaming.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
index 04676e1..609f669 100644
--- a/tests/test_parquet_streaming.py
+++ b/tests/test_parquet_streaming.py
@@ -266,6 +266,32 @@ def fail_load(self):
     assert candidate["confirmation_scope"] == "bounded_row_sample"
 
 
+def test_public_anomalies_stream_numeric_projection_without_calling_load(tmp_path, monkeypatch):
+    path = tmp_path / "anomalies.parquet"
+    frame = _write_parquet(path, rows=12_000)
+
+    def fail_load(self):
+        raise AssertionError("anomalies must not materialize the complete Parquet file")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.anomalies(
+        path,
+        mode="quick",
+        max_columns=2,
+        top_k=5,
+    )
+
+    execution = result["execution"]
+    assert execution["full_materialization"] is False
+    assert execution["source_rows"] == len(frame)
+    assert execution["source_columns"] == 4
+    assert execution["projected_columns"] == 2
+    assert execution["sample_rows"] == 5_000
+    assert execution["sampled"] is True
+    assert execution["strategy"] == "streaming_evenly_spaced_numeric_projection"
+    assert result["source"]["supports_projection"] is True
+
+
 def test_plan_reads_only_bounded_parquet_sample_and_uses_true_shape(tmp_path, monkeypatch):
     path = tmp_path / "plan.parquet"
     frame = _write_parquet(path, rows=12_000)

From 704175e1eb1c33c39080429c11d0344dc6d2039a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:18:01 +0530
Subject: [PATCH 205/439] feat: add bounded streaming target diagnostics

---
 src/framevitals/streaming_target.py | 101 ++++++++++++++++++++++++++++
 1 file changed, 101 insertions(+)
 create mode 100644 src/framevitals/streaming_target.py

diff --git a/src/framevitals/streaming_target.py b/src/framevitals/streaming_target.py
new file mode 100644
index 0000000..5b2834b
--- /dev/null
+++ b/src/framevitals/streaming_target.py
@@ -0,0 +1,101 @@
+"""Target-aware diagnostics for streaming dataset sources."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from framevitals.column_roles import infer_column_roles
+from framevitals.execution import derive_execution_budget
+from framevitals.sources import StreamingDatasetSource
+from framevitals.streaming_profile import sample_streaming_source
+from framevitals.target_intelligence import run_target_intelligence
+
+
+def _column_names(source: StreamingDatasetSource) -> list[str]:
+    schema_method = getattr(source, "schema", None)
+    if callable(schema_method):
+        schema = schema_method()
+        names = getattr(schema, "names", None)
+        if names is not None:
+            return [str(name) for name in names]
+
+    first_batch = next(source.iter_batches(batch_size=1), None)
+    if first_batch is None:
+        return []
+    schema = getattr(first_batch, "schema", None)
+    names = getattr(schema, "names", None)
+    if names is None:
+        return []
+    return [str(name) for name in names]
+
+
+def run_streaming_target_analysis(
+    source: StreamingDatasetSource,
+    *,
+    target: str,
+    max_columns: int = 200,
+) -> dict[str, Any]:
+    """Run target diagnostics on a deterministic bounded source projection.
+
+    The target is always retained. Feature columns are kept in source order up
+    to ``max_columns - 1``. This prevents an ultra-wide source from turning a
+    target-only diagnostic request into an unbounded pandas materialization.
+    """
+    if max_columns < 2:
+        raise ValueError("max_columns must be at least 2.")
+
+    metadata = source.inspect()
+    if metadata.rows is None or metadata.columns is None:
+        raise ValueError("Streaming target analysis requires source shape metadata.")
+    source_rows = int(metadata.rows)
+    source_columns = int(metadata.columns)
+
+    names = _column_names(source)
+    if target not in names:
+        raise ValueError(f"Target column not found: {target}")
+
+    features = [name for name in names if name != target]
+    selected_features = features[: max_columns - 1]
+    selected_set = set(selected_features)
+    selected_columns = [
+        name for name in names if name == target or name in selected_set
+    ]
+
+    budget = derive_execution_budget(
+        source_rows,
+        source_columns,
+        mode="standard",
+    )
+    sample_limit = max(100, int(budget.pair_sample_rows))
+    sample = sample_streaming_source(
+        source,
+        sample_rows=sample_limit,
+        columns=selected_columns,
+    )
+
+    column_roles = infer_column_roles(sample)
+    payload = run_target_intelligence(
+        sample,
+        target_column=target,
+        column_roles=column_roles,
+    )
+    payload["execution"] = {
+        "scope": (
+            "bounded_row_and_column_sample"
+            if len(sample) < source_rows or len(selected_columns) < source_columns
+            else "full_source"
+        ),
+        "full_materialization": False,
+        "source_rows": source_rows,
+        "source_columns": source_columns,
+        "sample_rows": int(len(sample)),
+        "sampled_rows": bool(len(sample) < source_rows),
+        "selected_columns": int(len(selected_columns)),
+        "feature_columns_considered": int(len(selected_features)),
+        "feature_columns_available": int(len(features)),
+        "columns_truncated": bool(len(selected_columns) < source_columns),
+        "strategy": "streaming_evenly_spaced_rows_with_deterministic_feature_projection",
+        "target_always_retained": True,
+    }
+    payload["source"] = metadata.to_dict()
+    return payload

From 33486f34e98786a0d0a4a5004b23cd794aee87bb Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:18:36 +0530
Subject: [PATCH 206/439] feat: stream bounded target diagnostics for Arrow
 sources

---
 src/framevitals/focused.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index 03952aa..c4f3ec5 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -395,10 +395,18 @@ def target_analysis(
     *,
     target: str,
 ) -> dict[str, Any]:
+    source = resolve_source(data)
+    metadata = source.inspect()
+    if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
+        from framevitals.streaming_target import run_streaming_target_analysis
+
+        payload = run_streaming_target_analysis(source, target=target)
+        return _named(payload, metadata.name)
+
     from framevitals.column_roles import infer_column_roles
     from framevitals.target_intelligence import run_target_intelligence
 
-    dataframe, source_name = _load(data)
+    dataframe = source.load()
     if target not in dataframe.columns:
         raise ValueError(f"Target column not found: {target}")
     column_roles = infer_column_roles(dataframe)
@@ -407,4 +415,4 @@ def target_analysis(
         target_column=target,
         column_roles=column_roles,
     )
-    return _named(payload, source_name)
+    return _named(payload, metadata.name)

From b9e6660eaa75b44f0756f4995c846b96e53403aa Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:19:16 +0530
Subject: [PATCH 207/439] feat: route installed CLI analyze through source
 dispatcher

---
 src/framevitals/cli_entry.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 src/framevitals/cli_entry.py

diff --git a/src/framevitals/cli_entry.py b/src/framevitals/cli_entry.py
new file mode 100644
index 0000000..e8c857e
--- /dev/null
+++ b/src/framevitals/cli_entry.py
@@ -0,0 +1,18 @@
+"""Installed CLI entrypoint with source-aware analysis routing.
+
+The legacy CLI parser remains the single command implementation. Before it
+runs, this shim replaces only ``framevitals.api.analyze`` with the newer source
+dispatcher so ``framevitals analyze`` and ``framevitals.analyze`` share the same
+streaming behavior without duplicating CLI parsing or output code.
+"""
+
+from __future__ import annotations
+
+
+def main() -> int:
+    from framevitals import api as legacy_api
+    from framevitals.analysis_api import analyze
+    from framevitals.cli import main as legacy_main
+
+    legacy_api.analyze = analyze
+    return legacy_main()

From 07f9df87b25ae2960c0c98687a898fd3abe991e1 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:19:29 +0530
Subject: [PATCH 208/439] build: use source-aware CLI entrypoint

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index cc02a7a..6224783 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -83,7 +83,7 @@ dev = [
 ]
 
 [project.scripts]
-framevitals = "framevitals.cli:main"
+framevitals = "framevitals.cli_entry:main"
 
 [project.urls]
 Documentation = "https://github.com/parthdongre/FrameVitals#readme"

From fa7c0031b167d04177a5170e59f0ba83761cbc11 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:19:33 +0530
Subject: [PATCH 209/439] feat: bound drift comparison for streaming sources

---
 src/framevitals/operations.py | 189 +++++++++++++++++++++++++++-------
 1 file changed, 152 insertions(+), 37 deletions(-)

diff --git a/src/framevitals/operations.py b/src/framevitals/operations.py
index 1b3887c..67ff8c9 100644
--- a/src/framevitals/operations.py
+++ b/src/framevitals/operations.py
@@ -21,40 +21,97 @@
 from framevitals.contracts import infer_contract as _infer_contract
 from framevitals.contracts import validate_contract
 from framevitals.drift_analysis import compare_datasets, severity_at_least
-from framevitals.loader import load_dataset
 from framevitals.profiler import build_profile
 from framevitals.quality_results import DriftResult, GateResult, ValidationResult
+from framevitals.sources import DatasetMetadata, StreamingDatasetSource, resolve_source
 
 
 DataInput = str | Path | pd.DataFrame
 _DRIFT_SEVERITY_RANK = {"stable": 0, "minor": 1, "moderate": 2, "severe": 3}
+_DRIFT_SAMPLE_ROWS = 50_000
 
 
-def _validated_path(value: str | Path, *, label: str) -> Path:
-    path = Path(value)
-    if not path.exists():
-        raise FileNotFoundError(f"{label} not found: {path}")
-    if not path.is_file():
-        raise ValueError(f"Expected a file for {label.lower()}, got: {path}")
-    return path
+def _resolve_input(value: DataInput, *, label: str):
+    try:
+        source = resolve_source(value)
+        metadata = source.inspect()
+    except (TypeError, ValueError, FileNotFoundError) as exc:
+        if label == "Dataset":
+            raise
+        message = str(exc).replace("Dataset", label, 1)
+        raise type(exc)(message) from exc
+
+    if metadata.rows == 0:
+        if metadata.kind == "memory":
+            raise ValueError(f"{label} DataFrame is empty.")
+        raise ValueError(f"{label} dataset is empty: {metadata.name}")
+    return source, metadata
 
 
 def _load_input(value: DataInput, *, label: str) -> tuple[pd.DataFrame, str]:
-    if isinstance(value, pd.DataFrame):
-        if value.empty:
+    source, metadata = _resolve_input(value, label=label)
+    dataframe = source.load()
+    if dataframe.empty:
+        if metadata.kind == "memory":
             raise ValueError(f"{label} DataFrame is empty.")
-        return value.copy(), "<dataframe>"
+        raise ValueError(f"{label} dataset is empty: {metadata.name}")
+    return dataframe, metadata.name
 
-    if isinstance(value, (str, Path)):
-        path = _validated_path(value, label=label)
-        dataframe = load_dataset(path)
-        if dataframe.empty:
-            raise ValueError(f"{label} dataset is empty: {path}")
-        return dataframe, path.name
 
-    raise TypeError(
-        f"{label} must be a pandas DataFrame or a path to a supported dataset."
-    )
+def _comparison_frame(
+    source,
+    metadata: DatasetMetadata,
+    *,
+    sample_rows: int = _DRIFT_SAMPLE_ROWS,
+) -> tuple[pd.DataFrame, dict[str, Any]]:
+    """Return drift input plus transparent source/materialization metadata."""
+    if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
+        from framevitals.streaming_profile import sample_streaming_source
+
+        if metadata.rows is None:
+            raise ValueError("Streaming drift comparison requires a source row count.")
+        sample = sample_streaming_source(source, sample_rows=sample_rows)
+        source_rows = int(metadata.rows)
+        sampled = len(sample) < source_rows
+        return sample, {
+            "source_rows": source_rows,
+            "source_columns": int(metadata.columns or len(sample.columns)),
+            "sample_rows": int(len(sample)),
+            "sampled": sampled,
+            "strategy": (
+                "streaming_evenly_spaced_global_rows"
+                if sampled
+                else "full_stream_via_batches"
+            ),
+            "full_materialization": False,
+            "source": metadata.to_dict(),
+        }
+
+    dataframe = source.load()
+    if dataframe.empty:
+        raise ValueError(f"Dataset is empty: {metadata.name}")
+    return dataframe, {
+        "source_rows": int(len(dataframe)),
+        "source_columns": int(len(dataframe.columns)),
+        "sample_rows": int(len(dataframe)),
+        "sampled": False,
+        "strategy": "full_input",
+        "full_materialization": bool(metadata.kind == "file"),
+        "source": metadata.to_dict(),
+    }
+
+
+def _true_shape(execution: Mapping[str, Any]) -> list[int]:
+    return [
+        int(execution.get("source_rows", 0)),
+        int(execution.get("source_columns", 0)),
+    ]
+
+
+def _row_count_change_percent(reference_rows: int, current_rows: int) -> float | None:
+    if reference_rows <= 0:
+        return None
+    return round((current_rows - reference_rows) / reference_rows * 100, 4)
 
 
 def plan_cleaning(data: DataInput) -> CleaningPlan:
@@ -82,20 +139,57 @@ def compare(
     columns: list[str] | None = None,
     max_columns: int = 30,
 ) -> DriftResult:
-    """Compare reference/current datasets without executing the EDA pipeline."""
+    """Compare datasets, bounding value-distribution work for streaming sources."""
     if max_columns < 1:
         raise ValueError("max_columns must be at least 1.")
 
-    reference_df, reference_name = _load_input(reference, label="Reference")
-    current_df, current_name = _load_input(current, label="Current")
+    reference_source, reference_metadata = _resolve_input(reference, label="Reference")
+    current_source, current_metadata = _resolve_input(current, label="Current")
+    reference_df, reference_execution = _comparison_frame(
+        reference_source,
+        reference_metadata,
+    )
+    current_df, current_execution = _comparison_frame(
+        current_source,
+        current_metadata,
+    )
+
     payload = compare_datasets(
         reference_df,
         current_df,
         columns=columns,
         max_columns=max_columns,
     )
-    payload["reference_name"] = reference_name
-    payload["current_name"] = current_name
+    payload["reference_name"] = reference_metadata.name
+    payload["current_name"] = current_metadata.name
+    payload["ref_shape"] = _true_shape(reference_execution)
+    payload["cur_shape"] = _true_shape(current_execution)
+    payload["row_count_change_percent"] = _row_count_change_percent(
+        reference_execution["source_rows"],
+        current_execution["source_rows"],
+    )
+
+    any_sampled = bool(
+        reference_execution["sampled"] or current_execution["sampled"]
+    )
+    payload["execution"] = {
+        "method": "bounded_source_compare" if any_sampled else "full_compare",
+        "sample_limit_rows_per_source": _DRIFT_SAMPLE_ROWS,
+        "full_materialization": bool(
+            reference_execution["full_materialization"]
+            or current_execution["full_materialization"]
+        ),
+        "reference": reference_execution,
+        "current": current_execution,
+        "components": {
+            "source_shape": "exact",
+            "schema_columns": "exact",
+            "value_distributions": (
+                "bounded_row_sample" if any_sampled else "full_input"
+            ),
+            "missingness": "bounded_row_sample" if any_sampled else "full_input",
+        },
+    }
     return DriftResult(payload)
 
 
@@ -129,9 +223,21 @@ def validate(
     contract: Mapping[str, Any],
 ) -> ValidationResult:
     """Validate a dataset against an inferred or explicit contract."""
-    dataframe, source_name = _load_input(data, label="Dataset")
+    source, metadata = _resolve_input(data, label="Dataset")
+    dataframe = source.load()
+    if dataframe.empty:
+        raise ValueError(f"Dataset is empty: {metadata.name}")
     payload = validate_contract(dataframe, contract)
-    payload["dataset_name"] = source_name
+    payload["dataset_name"] = metadata.name
+    payload["execution"] = {
+        "method": "exact_contract_validation",
+        "full_materialization": bool(metadata.kind == "file"),
+        "source": metadata.to_dict(),
+        "reason": (
+            "Contract validation remains exact; uniqueness, allowed-value, and bound "
+            "constraints are not silently downgraded to sampled checks."
+        ),
+    }
     return ValidationResult(payload)
 
 
@@ -146,7 +252,7 @@ def gate(
     drift_fail_on: str = "severe",
     fail_on_validation_warning: bool = False,
 ) -> GateResult:
-    """Combine contract/drift checks into one CI-friendly quality verdict."""
+    """Combine exact contract checks and bounded drift into one CI-friendly verdict."""
     if reference is None and contract is None:
         raise ValueError("gate requires at least one of reference= or contract=.")
     if drift_warn_on not in _DRIFT_SEVERITY_RANK:
@@ -158,7 +264,8 @@ def gate(
     if max_columns < 1:
         raise ValueError("max_columns must be at least 1.")
 
-    current_df, current_name = _load_input(current, label="Current")
+    _, current_metadata = _resolve_input(current, label="Current")
+    current_name = current_metadata.name
     checks: dict[str, Any] = {}
     reasons: list[str] = []
     status = "pass"
@@ -173,8 +280,7 @@ def fail() -> None:
         status = "fail"
 
     if contract is not None:
-        validation_payload = validate_contract(current_df, contract)
-        validation_payload["dataset_name"] = current_name
+        validation_payload = validate(current, contract)
         checks["validation"] = validation_payload
 
         validation_status = validation_payload.get("status")
@@ -194,15 +300,12 @@ def fail() -> None:
                 reasons.append(f"Contract validation produced {warning_count} warning(s).")
 
     if reference is not None:
-        reference_df, reference_name = _load_input(reference, label="Reference")
-        drift_payload = compare_datasets(
-            reference_df,
-            current_df,
+        drift_payload = compare(
+            reference,
+            current,
             columns=columns,
             max_columns=max_columns,
         )
-        drift_payload["reference_name"] = reference_name
-        drift_payload["current_name"] = current_name
         checks["drift"] = drift_payload
 
         if not drift_payload.get("available"):
@@ -250,4 +353,16 @@ def fail() -> None:
         },
         "reasons": reasons,
         "checks": checks,
+        "execution": {
+            "validation": (
+                checks.get("validation", {}).get("execution")
+                if isinstance(checks.get("validation"), Mapping)
+                else None
+            ),
+            "drift": (
+                checks.get("drift", {}).get("execution")
+                if isinstance(checks.get("drift"), Mapping)
+                else None
+            ),
+        },
     })

From 186ce54ca8073eb38fe20c9d094b22e68cd3793f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:19:47 +0530
Subject: [PATCH 210/439] test: verify installed analyze command uses streaming
 dispatcher

---
 tests/test_cli_entry_streaming.py | 52 +++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 tests/test_cli_entry_streaming.py

diff --git a/tests/test_cli_entry_streaming.py b/tests/test_cli_entry_streaming.py
new file mode 100644
index 0000000..2ea9c07
--- /dev/null
+++ b/tests/test_cli_entry_streaming.py
@@ -0,0 +1,52 @@
+import json
+
+import numpy as np
+import pandas as pd
+import pytest
+
+pa = pytest.importorskip("pyarrow")
+pq = pytest.importorskip("pyarrow.parquet")
+
+from framevitals.cli_entry import main
+from framevitals.sources import ParquetSource
+
+
+def test_installed_cli_analyze_streams_parquet(tmp_path, monkeypatch, capsys):
+    path = tmp_path / "cli-stream.parquet"
+    frame = pd.DataFrame({
+        "value": np.arange(12_000, dtype=np.float64),
+        "other": np.arange(12_000, dtype=np.float64) * 2.0,
+        "group": [f"g-{index % 5}" for index in range(12_000)],
+    })
+    pq.write_table(
+        pa.Table.from_pandas(frame, preserve_index=False),
+        path,
+        row_group_size=777,
+    )
+
+    def fail_load(self):
+        raise AssertionError("installed CLI analyze must not materialize full Parquet")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "analyze",
+            str(path),
+            "--mode",
+            "quick",
+            "--no-artifacts",
+            "--workers",
+            "1",
+            "--format",
+            "json",
+        ],
+    )
+
+    assert main() == 0
+    payload = json.loads(capsys.readouterr().out)
+    assert payload["profile"]["shape"] == {"rows": len(frame), "columns": 3}
+    assert payload["execution"]["streaming"]["enabled"] is True
+    assert payload["execution"]["streaming"]["full_materialization"] is False
+    assert payload["execution"]["streaming"]["working_sample_rows"] == 5_000

From fb4943834fcf8e6cb5dc6738b29a300c2119256b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:20:00 +0530
Subject: [PATCH 211/439] test: cover bounded streaming drift and gates

---
 tests/test_streaming_drift.py | 82 +++++++++++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 tests/test_streaming_drift.py

diff --git a/tests/test_streaming_drift.py b/tests/test_streaming_drift.py
new file mode 100644
index 0000000..7453d9e
--- /dev/null
+++ b/tests/test_streaming_drift.py
@@ -0,0 +1,82 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+pa = pytest.importorskip("pyarrow")
+pq = pytest.importorskip("pyarrow.parquet")
+
+import framevitals
+from framevitals.sources import ParquetSource
+
+
+def _write_drift_parquet(path, *, rows: int, shift: float = 0.0) -> pd.DataFrame:
+    frame = pd.DataFrame({
+        "value": np.arange(rows, dtype=np.float64) + shift,
+        "segment": [f"s-{index % 5}" for index in range(rows)],
+    })
+    frame.loc[::211, "value"] = np.nan
+    pq.write_table(
+        pa.Table.from_pandas(frame, preserve_index=False),
+        path,
+        row_group_size=777,
+    )
+    return frame
+
+
+def test_compare_streams_large_parquet_and_reports_true_source_shapes(
+    tmp_path,
+    monkeypatch,
+):
+    reference_path = tmp_path / "reference.parquet"
+    current_path = tmp_path / "current.parquet"
+    reference = _write_drift_parquet(reference_path, rows=60_000)
+    current = _write_drift_parquet(current_path, rows=72_000, shift=250.0)
+
+    def fail_load(self):
+        raise AssertionError("drift comparison must not fully materialize Parquet inputs")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.compare(reference_path, current_path)
+
+    assert result["available"] is True
+    assert result["reference_name"] == "reference.parquet"
+    assert result["current_name"] == "current.parquet"
+    assert result["ref_shape"] == [len(reference), len(reference.columns)]
+    assert result["cur_shape"] == [len(current), len(current.columns)]
+    assert result["row_count_change_percent"] == pytest.approx(20.0)
+
+    execution = result["execution"]
+    assert execution["method"] == "bounded_source_compare"
+    assert execution["full_materialization"] is False
+    assert execution["sample_limit_rows_per_source"] == 50_000
+    assert execution["reference"]["source_rows"] == len(reference)
+    assert execution["reference"]["sample_rows"] == 50_000
+    assert execution["reference"]["sampled"] is True
+    assert execution["current"]["source_rows"] == len(current)
+    assert execution["current"]["sample_rows"] == 50_000
+    assert execution["current"]["sampled"] is True
+    assert execution["components"]["source_shape"] == "exact"
+    assert execution["components"]["value_distributions"] == "bounded_row_sample"
+    assert execution["components"]["missingness"] == "bounded_row_sample"
+
+
+def test_reference_only_gate_reuses_streaming_drift_path(tmp_path, monkeypatch):
+    reference_path = tmp_path / "gate-reference.parquet"
+    current_path = tmp_path / "gate-current.parquet"
+    _write_drift_parquet(reference_path, rows=12_000)
+    _write_drift_parquet(current_path, rows=12_000, shift=25.0)
+
+    def fail_load(self):
+        raise AssertionError("reference-only gate must not materialize Parquet inputs")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    result = framevitals.gate(current_path, reference=reference_path)
+
+    assert result["checks_run"] == ["drift"]
+    assert result["execution"]["validation"] is None
+    drift_execution = result["execution"]["drift"]
+    assert drift_execution["full_materialization"] is False
+    assert drift_execution["reference"]["source_rows"] == 12_000
+    assert drift_execution["current"]["source_rows"] == 12_000
+    assert drift_execution["reference"]["strategy"] == "full_stream_via_batches"
+    assert drift_execution["current"]["strategy"] == "full_stream_via_batches"

From b197eaec2a0ee26d24c50408bc761dcb2e73f170 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:20:25 +0530
Subject: [PATCH 212/439] ci: exercise streaming drift regression

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4e921e6..3362ba5 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -230,7 +230,7 @@ jobs:
       - name: Run streaming Parquet tests without native extension
         env:
           FRAMEVITALS_BACKEND: numpy
-        run: pytest -v tests/test_parquet_streaming.py tests/test_statistics_streaming.py
+        run: pytest -v tests/test_parquet_streaming.py tests/test_statistics_streaming.py tests/test_streaming_drift.py
 
   optional-features:
     name: optional features (Python 3.12)

From 8d6674b85a666257b31c2cebef75ebe7b3e51e58 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:20:49 +0530
Subject: [PATCH 213/439] fix: include Arrow capability in all extra

---
 pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyproject.toml b/pyproject.toml
index 6224783..bc38531 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -72,6 +72,7 @@ all = [
     "flask>=3.0",
     "gunicorn>=22.0",
     "werkzeug>=3.0",
+    "pyarrow>=25.0,<26",
 ]
 dev = [
     "pytest>=8.2",

From 5eb94cf95317bf192cf5c4fb14e3ca1295db4b32 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:21:56 +0530
Subject: [PATCH 214/439] feat: stream CSV and TSV sources with Arrow

---
 src/framevitals/sources.py | 159 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 157 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/sources.py b/src/framevitals/sources.py
index 667cc3d..b37d824 100644
--- a/src/framevitals/sources.py
+++ b/src/framevitals/sources.py
@@ -8,7 +8,7 @@
 from __future__ import annotations
 
 from collections.abc import Iterator, Sequence
-from dataclasses import asdict, dataclass
+from dataclasses import asdict, dataclass, field
 from pathlib import Path
 from typing import Any, Protocol, runtime_checkable
 
@@ -108,6 +108,156 @@ def load(self) -> pd.DataFrame:
         return dataframe
 
 
+@dataclass(slots=True)
+class DelimitedTextSource:
+    """CSV/TSV source that streams through optional PyArrow when available.
+
+    Without the Arrow extra this source deliberately advertises itself as
+    non-streaming and falls back to the existing pandas loader. With Arrow it
+    performs one bounded-memory metadata scan to obtain an exact row count,
+    caches that metadata, and then yields projected Arrow record batches.
+    """
+
+    path: Path
+    delimiter: str = ","
+    _metadata_cache: DatasetMetadata | None = field(default=None, init=False, repr=False)
+    _schema_cache: Any = field(default=None, init=False, repr=False)
+    _arrow_compatible: bool | None = field(default=None, init=False, repr=False)
+
+    @property
+    def format(self) -> str:
+        return "tsv" if self.delimiter == "\t" else "csv"
+
+    def _pyarrow_csv(self):
+        try:
+            import pyarrow.csv as pacsv
+        except ImportError:
+            return None
+        return pacsv
+
+    def _arrow_reader(self, *, columns: Sequence[str] | None = None):
+        pacsv = self._pyarrow_csv()
+        if pacsv is None:
+            raise ImportError(
+                "CSV/TSV streaming requires the optional Arrow capability. "
+                'Install it with: pip install "framevitals[arrow]"'
+            )
+
+        read_options = pacsv.ReadOptions(use_threads=True)
+        parse_options = pacsv.ParseOptions(delimiter=self.delimiter)
+        convert_options = pacsv.ConvertOptions(
+            include_columns=list(columns) if columns is not None else None,
+        )
+        return pacsv.open_csv(
+            str(self.path),
+            read_options=read_options,
+            parse_options=parse_options,
+            convert_options=convert_options,
+        )
+
+    def inspect(self) -> DatasetMetadata:
+        _validate_file_path(self.path)
+        if self._metadata_cache is not None:
+            return self._metadata_cache
+
+        if self._pyarrow_csv() is None:
+            self._arrow_compatible = False
+            self._metadata_cache = DatasetMetadata(
+                name=self.path.name,
+                kind="file",
+                format=self.format,
+                rows=None,
+                columns=None,
+                size_bytes=int(self.path.stat().st_size),
+                materialized=False,
+                supports_projection=False,
+                supports_streaming=False,
+            )
+            return self._metadata_cache
+
+        try:
+            reader = self._arrow_reader()
+            schema = reader.schema
+            rows = 0
+            while True:
+                try:
+                    batch = reader.read_next_batch()
+                except StopIteration:
+                    break
+                rows += int(batch.num_rows)
+        except Exception:
+            # Preserve existing CSV/TSV compatibility if Arrow cannot parse a
+            # file that the pandas loader may still understand.
+            self._arrow_compatible = False
+            self._metadata_cache = DatasetMetadata(
+                name=self.path.name,
+                kind="file",
+                format=self.format,
+                rows=None,
+                columns=None,
+                size_bytes=int(self.path.stat().st_size),
+                materialized=False,
+                supports_projection=False,
+                supports_streaming=False,
+            )
+            return self._metadata_cache
+
+        self._arrow_compatible = True
+        self._schema_cache = schema
+        self._metadata_cache = DatasetMetadata(
+            name=self.path.name,
+            kind="file",
+            format=self.format,
+            rows=int(rows),
+            columns=int(len(schema)),
+            size_bytes=int(self.path.stat().st_size),
+            materialized=False,
+            supports_projection=True,
+            supports_streaming=True,
+        )
+        return self._metadata_cache
+
+    def schema(self):
+        """Return the inferred Arrow schema without materializing row data."""
+        metadata = self.inspect()
+        if not metadata.supports_streaming:
+            raise TypeError(
+                f"{self.format.upper()} source is not Arrow-streamable: {self.path}"
+            )
+        if self._schema_cache is None:
+            self._schema_cache = self._arrow_reader().schema
+        return self._schema_cache
+
+    def iter_batches(
+        self,
+        *,
+        batch_size: int = 65_536,
+        columns: Sequence[str] | None = None,
+    ) -> Iterator[Any]:
+        if batch_size < 1:
+            raise ValueError("batch_size must be at least 1.")
+        metadata = self.inspect()
+        if not metadata.supports_streaming:
+            raise TypeError(
+                f"{self.format.upper()} source cannot stream without a compatible Arrow reader."
+            )
+
+        reader = self._arrow_reader(columns=columns)
+        while True:
+            try:
+                batch = reader.read_next_batch()
+            except StopIteration:
+                break
+            for offset in range(0, int(batch.num_rows), int(batch_size)):
+                yield batch.slice(offset, min(int(batch_size), int(batch.num_rows) - offset))
+
+    def load(self) -> pd.DataFrame:
+        dataframe = load_dataset(self.path)
+        if dataframe.empty:
+            raise ValueError(f"Dataset is empty: {self.path}")
+        return dataframe
+
+
 @dataclass(slots=True)
 class ParquetSource:
     """Projection-aware, streaming Parquet source backed by optional PyArrow."""
@@ -181,8 +331,13 @@ def resolve_source(data: str | Path | pd.DataFrame | DatasetSource) -> DatasetSo
         return PandasSource(data)
     if isinstance(data, (str, Path)):
         path = Path(data)
-        if path.suffix.lower() == ".parquet":
+        suffix = path.suffix.lower()
+        if suffix == ".parquet":
             return ParquetSource(path)
+        if suffix == ".csv":
+            return DelimitedTextSource(path, delimiter=",")
+        if suffix == ".tsv":
+            return DelimitedTextSource(path, delimiter="\t")
         return FileSource(path)
     if isinstance(data, DatasetSource):
         return data

From dff59468c0e52396b768fc9f1d0a1764c0ec43f0 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:22:13 +0530
Subject: [PATCH 215/439] test: cover Arrow CSV and TSV streaming

---
 tests/test_csv_streaming.py | 96 +++++++++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 tests/test_csv_streaming.py

diff --git a/tests/test_csv_streaming.py b/tests/test_csv_streaming.py
new file mode 100644
index 0000000..1918641
--- /dev/null
+++ b/tests/test_csv_streaming.py
@@ -0,0 +1,96 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+pytest.importorskip("pyarrow")
+
+import framevitals
+from framevitals.sources import DelimitedTextSource, resolve_source
+
+
+def _frame(rows: int) -> pd.DataFrame:
+    frame = pd.DataFrame({
+        "value": np.arange(rows, dtype=np.float64),
+        "other": np.arange(rows, dtype=np.float64) * 1.5,
+        "group": [f"g-{index % 6}" for index in range(rows)],
+    })
+    frame.loc[::137, "value"] = np.nan
+    return frame
+
+
+def test_csv_source_exposes_exact_streaming_metadata_and_projection(tmp_path):
+    path = tmp_path / "dataset.csv"
+    frame = _frame(4_000)
+    frame.to_csv(path, index=False)
+
+    source = resolve_source(path)
+    assert isinstance(source, DelimitedTextSource)
+
+    metadata = source.inspect()
+    assert metadata.format == "csv"
+    assert metadata.rows == len(frame)
+    assert metadata.columns == len(frame.columns)
+    assert metadata.materialized is False
+    assert metadata.supports_projection is True
+    assert metadata.supports_streaming is True
+
+    batches = list(source.iter_batches(batch_size=700, columns=["value", "group"]))
+    assert sum(batch.num_rows for batch in batches) == len(frame)
+    assert max(batch.num_rows for batch in batches) <= 700
+    assert batches[0].schema.names == ["value", "group"]
+
+
+def test_public_profile_streams_csv_without_calling_load(tmp_path, monkeypatch):
+    path = tmp_path / "stream.csv"
+    frame = _frame(12_000)
+    frame.to_csv(path, index=False)
+
+    def fail_load(self):
+        raise AssertionError("streaming CSV profile must not materialize the complete file")
+
+    monkeypatch.setattr(DelimitedTextSource, "load", fail_load)
+    result = framevitals.profile(path)
+
+    assert result["dataset_name"] == "stream.csv"
+    assert result["shape"] == {"rows": len(frame), "columns": len(frame.columns)}
+    assert result["streaming_metadata"]["enabled"] is True
+    assert result["streaming_metadata"]["full_materialization"] is False
+    assert result["source_metadata"]["format"] == "csv"
+    assert result["source_metadata"]["supports_streaming"] is True
+    assert result["missing_counts"]["value"] == int(frame["value"].isna().sum())
+
+
+def test_public_analyze_dispatches_csv_through_streaming_pipeline(tmp_path, monkeypatch):
+    path = tmp_path / "analyze.csv"
+    frame = _frame(6_000)
+    frame.to_csv(path, index=False)
+
+    def fail_load(self):
+        raise AssertionError("streaming CSV analysis must not materialize the complete file")
+
+    monkeypatch.setattr(DelimitedTextSource, "load", fail_load)
+    result = framevitals.analyze(path, mode="quick", artifacts=False, workers=1)
+
+    assert result["filename"] == "analyze.csv"
+    assert result["profile"]["shape"] == {"rows": len(frame), "columns": len(frame.columns)}
+    assert result["execution"]["streaming"]["enabled"] is True
+    assert result["execution"]["streaming"]["full_materialization"] is False
+    assert result["execution"]["streaming"]["source_rows"] == len(frame)
+
+
+def test_tsv_source_streams_with_tab_delimiter(tmp_path):
+    path = tmp_path / "dataset.tsv"
+    frame = _frame(1_500)
+    frame.to_csv(path, index=False, sep="\t")
+
+    source = resolve_source(path)
+    assert isinstance(source, DelimitedTextSource)
+    metadata = source.inspect()
+
+    assert metadata.format == "tsv"
+    assert metadata.rows == len(frame)
+    assert metadata.columns == len(frame.columns)
+    assert metadata.supports_streaming is True
+    batch = next(source.iter_batches(batch_size=200, columns=["group"]))
+    assert batch.schema.names == ["group"]
+    assert batch.num_rows <= 200

From 396ea1c22d73667373878e39b9083aa7bdc5842e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:22:40 +0530
Subject: [PATCH 216/439] ci: validate CSV streaming and complete all extra

---
 .github/workflows/test.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3362ba5..0996dda 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -227,10 +227,10 @@ jobs:
       - name: Verify dependency consistency
         run: python -m pip check
 
-      - name: Run streaming Parquet tests without native extension
+      - name: Run streaming source tests without native extension
         env:
           FRAMEVITALS_BACKEND: numpy
-        run: pytest -v tests/test_parquet_streaming.py tests/test_statistics_streaming.py tests/test_streaming_drift.py
+        run: pytest -v tests/test_parquet_streaming.py tests/test_statistics_streaming.py tests/test_streaming_drift.py tests/test_csv_streaming.py
 
   optional-features:
     name: optional features (Python 3.12)
@@ -263,6 +263,7 @@ jobs:
           import flask
           import lightgbm
           import ollama
+          import pyarrow
           import pyod
           import shap
           import xgboost

From 8d74fdf2d7b337c9b4af900b70395fccfbb053c2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:24:44 +0530
Subject: [PATCH 217/439] fix: preserve FileSource compatibility for delimited
 inputs

---
 src/framevitals/sources.py | 45 ++++++++++----------------------------
 1 file changed, 11 insertions(+), 34 deletions(-)

diff --git a/src/framevitals/sources.py b/src/framevitals/sources.py
index b37d824..e387a43 100644
--- a/src/framevitals/sources.py
+++ b/src/framevitals/sources.py
@@ -109,16 +109,16 @@ def load(self) -> pd.DataFrame:
 
 
 @dataclass(slots=True)
-class DelimitedTextSource:
-    """CSV/TSV source that streams through optional PyArrow when available.
-
-    Without the Arrow extra this source deliberately advertises itself as
-    non-streaming and falls back to the existing pandas loader. With Arrow it
-    performs one bounded-memory metadata scan to obtain an exact row count,
-    caches that metadata, and then yields projected Arrow record batches.
+class DelimitedTextSource(FileSource):
+    """CSV/TSV file source with optional Arrow streaming acceleration.
+
+    This remains a :class:`FileSource` for compatibility. Without the Arrow
+    extra it advertises the same non-streaming behaviour as a normal file
+    source and falls back to the existing pandas loader. With Arrow it performs
+    one bounded-memory metadata scan to obtain an exact row count, caches that
+    metadata, and then yields projected Arrow record batches.
     """
 
-    path: Path
     delimiter: str = ","
     _metadata_cache: DatasetMetadata | None = field(default=None, init=False, repr=False)
     _schema_cache: Any = field(default=None, init=False, repr=False)
@@ -162,17 +162,7 @@ def inspect(self) -> DatasetMetadata:
 
         if self._pyarrow_csv() is None:
             self._arrow_compatible = False
-            self._metadata_cache = DatasetMetadata(
-                name=self.path.name,
-                kind="file",
-                format=self.format,
-                rows=None,
-                columns=None,
-                size_bytes=int(self.path.stat().st_size),
-                materialized=False,
-                supports_projection=False,
-                supports_streaming=False,
-            )
+            self._metadata_cache = super().inspect()
             return self._metadata_cache
 
         try:
@@ -189,17 +179,7 @@ def inspect(self) -> DatasetMetadata:
             # Preserve existing CSV/TSV compatibility if Arrow cannot parse a
             # file that the pandas loader may still understand.
             self._arrow_compatible = False
-            self._metadata_cache = DatasetMetadata(
-                name=self.path.name,
-                kind="file",
-                format=self.format,
-                rows=None,
-                columns=None,
-                size_bytes=int(self.path.stat().st_size),
-                materialized=False,
-                supports_projection=False,
-                supports_streaming=False,
-            )
+            self._metadata_cache = super().inspect()
             return self._metadata_cache
 
         self._arrow_compatible = True
@@ -252,10 +232,7 @@ def iter_batches(
                 yield batch.slice(offset, min(int(batch_size), int(batch.num_rows) - offset))
 
     def load(self) -> pd.DataFrame:
-        dataframe = load_dataset(self.path)
-        if dataframe.empty:
-            raise ValueError(f"Dataset is empty: {self.path}")
-        return dataframe
+        return super().load()
 
 
 @dataclass(slots=True)

From 315c19be187baa231acd2851fd3d6f29e1dcc749 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:25:45 +0530
Subject: [PATCH 218/439] refactor: route CLI through canonical public engines

---
 src/framevitals/cli.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/framevitals/cli.py b/src/framevitals/cli.py
index af1e241..a978c72 100644
--- a/src/framevitals/cli.py
+++ b/src/framevitals/cli.py
@@ -338,7 +338,7 @@ def main() -> int:
     args = parser.parse_args()
 
     if args.command == "analyze":
-        from framevitals.api import analyze
+        from framevitals.analysis_api import analyze
 
         report = analyze(
             args.file,
@@ -378,7 +378,7 @@ def main() -> int:
         return 0
 
     if args.command == "plan":
-        from framevitals.api import plan
+        from framevitals.planning_api import plan
 
         result = plan(
             args.file,
@@ -403,7 +403,7 @@ def main() -> int:
         return 0
 
     if args.command == "clean":
-        from framevitals.api import clean, plan_cleaning
+        from framevitals.operations import clean, plan_cleaning
         from framevitals.security import sanitize_csv_value
 
         cleaning_plan = plan_cleaning(args.file)
@@ -427,8 +427,8 @@ def main() -> int:
         return 0
 
     if args.command == "compare":
-        from framevitals.api import compare
         from framevitals.drift_analysis import severity_at_least
+        from framevitals.operations import compare
 
         columns = None
         if args.columns:
@@ -459,7 +459,7 @@ def main() -> int:
         return 0
 
     if args.command == "infer-contract":
-        from framevitals.api import infer_contract
+        from framevitals.operations import infer_contract
 
         report = infer_contract(
             args.file,
@@ -474,7 +474,7 @@ def main() -> int:
         return 0
 
     if args.command == "validate":
-        from framevitals.api import validate
+        from framevitals.operations import validate
 
         report = validate(
             args.file,

From f5cbe0e7763c38ef8dc076d556d695fba0bff6cc Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:25:56 +0530
Subject: [PATCH 219/439] refactor: retire CLI monkeypatch shim

---
 src/framevitals/cli_entry.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/framevitals/cli_entry.py b/src/framevitals/cli_entry.py
index e8c857e..ec6c099 100644
--- a/src/framevitals/cli_entry.py
+++ b/src/framevitals/cli_entry.py
@@ -1,18 +1,14 @@
-"""Installed CLI entrypoint with source-aware analysis routing.
+"""Backward-compatible CLI entrypoint.
 
-The legacy CLI parser remains the single command implementation. Before it
-runs, this shim replaces only ``framevitals.api.analyze`` with the newer source
-dispatcher so ``framevitals analyze`` and ``framevitals.analyze`` share the same
-streaming behavior without duplicating CLI parsing or output code.
+The installed console script now points directly to :mod:`framevitals.cli`.
+This module remains as a lightweight compatibility alias for callers that
+imported ``framevitals.cli_entry.main`` during the 0.x series.
 """
 
 from __future__ import annotations
 
 
 def main() -> int:
-    from framevitals import api as legacy_api
-    from framevitals.analysis_api import analyze
-    from framevitals.cli import main as legacy_main
+    from framevitals.cli import main as cli_main
 
-    legacy_api.analyze = analyze
-    return legacy_main()
+    return cli_main()

From 73f60ebc49d037f0ea66e8b24a3b6193c9a2d73d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:26:06 +0530
Subject: [PATCH 220/439] packaging: use canonical CLI entrypoint

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index bc38531..3420961 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -84,7 +84,7 @@ dev = [
 ]
 
 [project.scripts]
-framevitals = "framevitals.cli_entry:main"
+framevitals = "framevitals.cli:main"
 
 [project.urls]
 Documentation = "https://github.com/parthdongre/FrameVitals#readme"

From 76e0d682a58b37e7e3fd4a91467c3a2c6e5963a9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:26:53 +0530
Subject: [PATCH 221/439] refactor: collapse legacy API into canonical facade

---
 src/framevitals/api.py | 498 ++++++++++++-----------------------------
 1 file changed, 149 insertions(+), 349 deletions(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index 0e60409..cbe0bb1 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -1,104 +1,55 @@
+"""Backward-compatible public API facade.
+
+Historically this module contained a second eager implementation of most
+FrameVitals operations. The canonical implementations now live in focused,
+source-aware modules. Keeping this file as a thin lazy facade preserves imports
+such as ``from framevitals.api import analyze`` without maintaining two engines
+that can diverge in behavior, streaming support, or execution metadata.
+"""
+
 from __future__ import annotations
 
 from collections.abc import Mapping
-from pathlib import Path
-from typing import Any
-from uuid import uuid4
-
-import pandas as pd
-
-from framevitals.analysis_selector import select_analyses
-from framevitals.anomaly_ensemble import detect_anomalies_ensemble
-from framevitals.cleaning_plan import (
-    CleaningPlan,
-    apply_cleaning_plan,
-    infer_cleaning_plan,
-)
-from framevitals.column_roles import infer_column_roles, summarize_roles
-from framevitals.config import ConfigInput, VALID_MODULES, resolve_config
-from framevitals.contracts import infer_contract as _infer_contract
-from framevitals.contracts import validate_contract
-from framevitals.dataset_signals import detect_dataset_signals
-from framevitals.deep_statistics_v2 import run_deep_statistics_v2
-from framevitals.drift_analysis import compare_datasets, severity_at_least
-from framevitals.health_score import calculate_health_score
-from framevitals.loader import load_dataset
-from framevitals.ml_readiness import calculate_ml_readiness
-from framevitals.pipeline import run_full_analysis
-from framevitals.planning import AnalysisPlan
-from framevitals.profiler import build_profile
-from framevitals.quality_diagnostics import run_quality_diagnostics
-from framevitals.quality_results import DriftResult, GateResult, ValidationResult
-from framevitals.result import AnalysisResult
-from framevitals.target_intelligence import run_target_intelligence
-
-DataInput = str | Path | pd.DataFrame
-_DRIFT_SEVERITY_RANK = {"stable": 0, "minor": 1, "moderate": 2, "severe": 3}
-
-
-def _validated_path(value: str | Path, *, label: str = "Dataset") -> Path:
-    path = Path(value)
-    if not path.exists():
-        raise FileNotFoundError(f"{label} not found: {path}")
-    if not path.is_file():
-        raise ValueError(f"Expected a file for {label.lower()}, got: {path}")
-    return path
-
-
-def _load_input(value: DataInput, *, label: str) -> tuple[pd.DataFrame, str]:
-    if isinstance(value, pd.DataFrame):
-        if value.empty:
-            raise ValueError(f"{label} DataFrame is empty.")
-        return value.copy(), "<dataframe>"
-
-    if isinstance(value, (str, Path)):
-        path = _validated_path(value, label=label)
-        df = load_dataset(path)
-        if df.empty:
-            raise ValueError(f"{label} dataset is empty: {path}")
-        return df, path.name
-
-    raise TypeError(
-        f"{label} must be a pandas DataFrame or a path to a supported dataset."
-    )
+from typing import TYPE_CHECKING, Any
 
+if TYPE_CHECKING:
+    import pandas as pd
 
-def _with_dataset_name(payload: dict[str, Any], source_name: str) -> dict[str, Any]:
-    """Attach source identity without mutating an analysis module's own result."""
-    return {"dataset_name": source_name, **payload}
+    from framevitals.cleaning_plan import CleaningPlan
+    from framevitals.planning import AnalysisPlan
+    from framevitals.quality_results import DriftResult, GateResult, ValidationResult
+    from framevitals.result import AnalysisResult
+
+
+DataInput = Any
 
 
 def profile(data: DataInput) -> dict[str, Any]:
-    """Profile shape, types, missingness, cardinality, and basic summaries only."""
-    dataframe, source_name = _load_input(data, label="Dataset")
-    return _with_dataset_name(build_profile(dataframe), source_name)
+    """Profile a dataset through the canonical focused execution path."""
+    from framevitals.focused import profile as _profile
+
+    return _profile(data)
 
 
 def roles(data: DataInput) -> dict[str, Any]:
-    """Infer semantic/structural column roles without running the full pipeline."""
-    dataframe, source_name = _load_input(data, label="Dataset")
-    column_roles = infer_column_roles(dataframe)
-    return {
-        "dataset_name": source_name,
-        "columns": column_roles,
-        "summary": summarize_roles(column_roles),
-    }
+    """Infer column roles through the canonical focused execution path."""
+    from framevitals.focused import roles as _roles
+
+    return _roles(data)
 
 
 def health(data: DataInput) -> dict[str, Any]:
-    """Calculate the FrameVitals data-health score only."""
-    dataframe, source_name = _load_input(data, label="Dataset")
-    dataset_profile = build_profile(dataframe)
-    payload = calculate_health_score(dataframe, dataset_profile)
-    return _with_dataset_name(payload, source_name)
+    """Calculate dataset health through the canonical focused execution path."""
+    from framevitals.focused import health as _health
+
+    return _health(data)
 
 
 def ml_readiness(data: DataInput) -> dict[str, Any]:
-    """Calculate ML-readiness diagnostics only."""
-    dataframe, source_name = _load_input(data, label="Dataset")
-    dataset_profile = build_profile(dataframe)
-    payload = calculate_ml_readiness(dataframe, profile=dataset_profile)
-    return _with_dataset_name(payload, source_name)
+    """Calculate ML readiness through the canonical focused execution path."""
+    from framevitals.focused import ml_readiness as _ml_readiness
+
+    return _ml_readiness(data)
 
 
 def quality(
@@ -108,30 +59,27 @@ def quality(
     max_columns: int = 100,
     max_missingness_columns: int = 25,
 ) -> dict[str, Any]:
-    """Run practical deterministic data-quality diagnostics only."""
-    dataframe, source_name = _load_input(data, label="Dataset")
-    dataset_profile = build_profile(dataframe)
-    column_roles = infer_column_roles(dataframe)
-    payload = run_quality_diagnostics(
-        dataframe,
-        profile=dataset_profile,
-        column_roles=column_roles,
+    """Run deterministic quality diagnostics through the focused engine."""
+    from framevitals.focused import quality as _quality
+
+    return _quality(
+        data,
         max_sample_rows=max_sample_rows,
         max_columns=max_columns,
         max_missingness_columns=max_missingness_columns,
     )
-    return _with_dataset_name(payload, source_name)
 
 
 def statistics(
     data: DataInput,
     *,
     max_pairs: int = 20,
+    mode: str = "standard",
 ) -> dict[str, Any]:
-    """Run the deep statistical diagnostics layer only."""
-    dataframe, source_name = _load_input(data, label="Dataset")
-    payload = run_deep_statistics_v2(dataframe, max_pairs=max_pairs)
-    return _with_dataset_name(payload, source_name)
+    """Run bounded deep statistics through the focused engine."""
+    from framevitals.focused import statistics as _statistics
+
+    return _statistics(data, max_pairs=max_pairs, mode=mode)
 
 
 def anomalies(
@@ -141,35 +89,48 @@ def anomalies(
     threshold: float = 0.6,
     max_columns: int = 30,
     top_k: int = 25,
+    mode: str = "standard",
 ) -> dict[str, Any]:
-    """Run the tabular anomaly ensemble only."""
-    dataframe, source_name = _load_input(data, label="Dataset")
-    payload = detect_anomalies_ensemble(
-        dataframe,
+    """Run bounded anomaly diagnostics through the focused engine."""
+    from framevitals.focused import anomalies as _anomalies
+
+    return _anomalies(
+        data,
         contamination=contamination,
         threshold=threshold,
         max_columns=max_columns,
         top_k=top_k,
+        mode=mode,
     )
-    return _with_dataset_name(payload, source_name)
 
 
-def target_analysis(
+def relationships(
     data: DataInput,
     *,
-    target: str,
+    max_sample_rows: int = 512,
+    projections: int = 64,
+    min_abs_correlation: float = 0.80,
+    max_candidate_pairs: int = 250_000,
+    max_edges_returned: int = 5_000,
 ) -> dict[str, Any]:
-    """Run target-quality, leakage, association, and split diagnostics only."""
-    dataframe, source_name = _load_input(data, label="Dataset")
-    if target not in dataframe.columns:
-        raise ValueError(f"Target column not found: {target}")
-    column_roles = infer_column_roles(dataframe)
-    payload = run_target_intelligence(
-        dataframe,
-        target_column=target,
-        column_roles=column_roles,
+    """Discover strong numeric relationships through the focused engine."""
+    from framevitals.focused import relationships as _relationships
+
+    return _relationships(
+        data,
+        max_sample_rows=max_sample_rows,
+        projections=projections,
+        min_abs_correlation=min_abs_correlation,
+        max_candidate_pairs=max_candidate_pairs,
+        max_edges_returned=max_edges_returned,
     )
-    return _with_dataset_name(payload, source_name)
+
+
+def target_analysis(data: DataInput, *, target: str) -> dict[str, Any]:
+    """Run target diagnostics through the focused source-aware engine."""
+    from framevitals.focused import target_analysis as _target_analysis
+
+    return _target_analysis(data, target=target)
 
 
 def analyze(
@@ -180,53 +141,22 @@ def analyze(
     artifacts: bool | None = None,
     workers: int | None = None,
     preset: str | None = None,
-    config: ConfigInput = None,
+    config: Any = None,
     disabled_modules: list[str] | tuple[str, ...] | None = None,
 ) -> AnalysisResult:
-    """Analyze a tabular dataset with the complete configured FrameVitals pipeline."""
-    resolved = resolve_config(
-        config,
-        preset=preset,
-        mode=mode,
+    """Analyze a dataset through the canonical source-aware dispatcher."""
+    from framevitals.analysis_api import analyze as _analyze
+
+    return _analyze(
+        data,
         target=target,
+        mode=mode,
         artifacts=artifacts,
         workers=workers,
+        preset=preset,
+        config=config,
         disabled_modules=disabled_modules,
     )
-    dataset_id = f"fv_{uuid4().hex[:12]}"
-
-    pipeline_kwargs = {
-        "dataset_id": dataset_id,
-        "original_filename": "<dataframe>",
-        "analysis_mode": resolved.mode,
-        "target_column": resolved.target,
-        "parallel_workers": resolved.workers,
-        "skip_ai": True,
-        "write_artifacts": resolved.artifacts,
-        "disabled_modules": resolved.disabled_modules,
-    }
-
-    if isinstance(data, pd.DataFrame):
-        if data.empty:
-            raise ValueError("Dataset DataFrame is empty.")
-        payload = run_full_analysis(
-            dataframe=data,
-            **pipeline_kwargs,
-        )
-    elif isinstance(data, (str, Path)):
-        path = _validated_path(data)
-        pipeline_kwargs["original_filename"] = path.name
-        payload = run_full_analysis(
-            file_path=path,
-            **pipeline_kwargs,
-        )
-    else:
-        raise TypeError(
-            "data must be a pandas DataFrame or a path to a supported dataset."
-        )
-
-    payload["config"] = resolved.to_dict()
-    return AnalysisResult(payload)
 
 
 def plan(
@@ -236,64 +166,28 @@ def plan(
     mode: str | None = None,
     workers: int | None = None,
     preset: str | None = None,
-    config: ConfigInput = None,
+    config: Any = None,
     disabled_modules: list[str] | tuple[str, ...] | None = None,
 ) -> AnalysisPlan:
-    """Preview which analyses FrameVitals considers applicable.
-
-    Planning performs loading, profiling, role inference, signal detection, and
-    selector evaluation only. It does not run the heavier model/statistics,
-    cleaning, chart, or AI stages of :func:`analyze`.
-    """
-    resolved = resolve_config(
-        config,
-        preset=preset,
-        mode=mode,
+    """Preview analysis execution through the canonical planning API."""
+    from framevitals.planning_api import plan as _plan
+
+    return _plan(
+        data,
         target=target,
+        mode=mode,
         workers=workers,
-        artifacts=False,
+        preset=preset,
+        config=config,
         disabled_modules=disabled_modules,
     )
-    dataframe, source_name = _load_input(data, label="Dataset")
-    dataset_profile = build_profile(dataframe)
-    column_roles = infer_column_roles(dataframe)
-    dataset_signals = detect_dataset_signals(
-        dataframe,
-        dataset_profile,
-        column_roles=column_roles,
-    )
-    selection = select_analyses(
-        signals=dataset_signals,
-        analysis_mode=resolved.mode,
-        target_column=resolved.target,
-    )
-    disabled = set(resolved.disabled_modules)
-    selection["execution_modules"] = {
-        "disabled": sorted(disabled),
-        "enabled": sorted(VALID_MODULES - disabled),
-    }
-
-    public_signals = {
-        key: value
-        for key, value in dataset_signals.items()
-        if key != "column_roles"
-    }
-    return AnalysisPlan({
-        "dataset_name": source_name,
-        "analysis_mode": resolved.mode,
-        "target": resolved.target,
-        "shape": dict(dataset_profile.get("shape", {})),
-        "config": resolved.to_dict(),
-        "signals": public_signals,
-        "selection": selection,
-    })
 
 
 def plan_cleaning(data: DataInput) -> CleaningPlan:
-    """Infer a conservative cleaning plan without modifying the input data."""
-    dataframe, _ = _load_input(data, label="Dataset")
-    dataset_profile = build_profile(dataframe)
-    return infer_cleaning_plan(dataframe, profile=dataset_profile)
+    """Infer a conservative cleaning plan through the operations layer."""
+    from framevitals.operations import plan_cleaning as _plan_cleaning
+
+    return _plan_cleaning(data)
 
 
 def clean(
@@ -301,14 +195,10 @@ def clean(
     *,
     plan: Mapping[str, Any] | None = None,
 ) -> pd.DataFrame:
-    """Return an explicitly cleaned copy of a dataset.
+    """Return an explicitly cleaned copy through the operations layer."""
+    from framevitals.operations import clean as _clean
 
-    If ``plan`` is omitted FrameVitals infers its conservative default plan.
-    The original DataFrame or source file is never modified in place.
-    """
-    dataframe, _ = _load_input(data, label="Dataset")
-    resolved_plan = plan if plan is not None else infer_cleaning_plan(dataframe)
-    return apply_cleaning_plan(dataframe, resolved_plan, copy=True)
+    return _clean(data, plan=plan)
 
 
 def compare(
@@ -318,22 +208,15 @@ def compare(
     columns: list[str] | None = None,
     max_columns: int = 30,
 ) -> DriftResult:
-    """Compare reference and current datasets for distribution and schema drift."""
-    if max_columns < 1:
-        raise ValueError("max_columns must be at least 1.")
+    """Compare reference/current data through the source-aware drift path."""
+    from framevitals.operations import compare as _compare
 
-    ref_df, ref_name = _load_input(reference, label="Reference")
-    cur_df, cur_name = _load_input(current, label="Current")
-
-    payload = compare_datasets(
-        ref_df,
-        cur_df,
+    return _compare(
+        reference,
+        current,
         columns=columns,
         max_columns=max_columns,
     )
-    payload["reference_name"] = ref_name
-    payload["current_name"] = cur_name
-    return DriftResult(payload)
 
 
 def infer_contract(
@@ -346,10 +229,11 @@ def infer_contract(
     min_unique_rows: int = 20,
     allow_extra_columns: bool = False,
 ) -> dict[str, Any]:
-    """Infer a JSON-serializable data contract from a reference dataset."""
-    dataframe, source_name = _load_input(data, label="Reference")
-    contract = _infer_contract(
-        dataframe,
+    """Infer a reusable data contract through the operations layer."""
+    from framevitals.operations import infer_contract as _infer_contract
+
+    return _infer_contract(
+        data,
         numeric_tolerance=numeric_tolerance,
         max_categories=max_categories,
         null_fraction_tolerance=null_fraction_tolerance,
@@ -357,19 +241,13 @@ def infer_contract(
         min_unique_rows=min_unique_rows,
         allow_extra_columns=allow_extra_columns,
     )
-    contract["reference_name"] = source_name
-    return contract
 
 
-def validate(
-    data: DataInput,
-    contract: Mapping[str, Any],
-) -> ValidationResult:
-    """Validate a dataset against an inferred or explicit data contract."""
-    dataframe, source_name = _load_input(data, label="Dataset")
-    payload = validate_contract(dataframe, contract)
-    payload["dataset_name"] = source_name
-    return ValidationResult(payload)
+def validate(data: DataInput, contract: Mapping[str, Any]) -> ValidationResult:
+    """Validate a dataset exactly through the operations layer."""
+    from framevitals.operations import validate as _validate
+
+    return _validate(data, contract)
 
 
 def gate(
@@ -383,115 +261,37 @@ def gate(
     drift_fail_on: str = "severe",
     fail_on_validation_warning: bool = False,
 ) -> GateResult:
-    """Run lightweight validation/drift checks and return one quality verdict.
-
-    At least one of ``reference`` or ``contract`` must be supplied. The current
-    dataset is loaded only once, and no full EDA/model pipeline is executed.
-    Validation failures always fail the gate. Warning-only validation results
-    warn by default and can be promoted to failures. Drift uses independent
-    warning/failure thresholds.
-    """
-    if reference is None and contract is None:
-        raise ValueError("gate requires at least one of reference= or contract=.")
-    if drift_warn_on not in _DRIFT_SEVERITY_RANK:
-        raise ValueError("drift_warn_on must be one of: stable, minor, moderate, severe.")
-    if drift_fail_on not in _DRIFT_SEVERITY_RANK:
-        raise ValueError("drift_fail_on must be one of: stable, minor, moderate, severe.")
-    if _DRIFT_SEVERITY_RANK[drift_warn_on] > _DRIFT_SEVERITY_RANK[drift_fail_on]:
-        raise ValueError("drift_warn_on cannot be more severe than drift_fail_on.")
-    if max_columns < 1:
-        raise ValueError("max_columns must be at least 1.")
-
-    current_df, current_name = _load_input(current, label="Current")
-    checks: dict[str, Any] = {}
-    reasons: list[str] = []
-    status = "pass"
-
-    def warn() -> None:
-        nonlocal status
-        if status == "pass":
-            status = "warn"
-
-    def fail() -> None:
-        nonlocal status
-        status = "fail"
-
-    if contract is not None:
-        validation = validate_contract(current_df, contract)
-        validation["dataset_name"] = current_name
-        checks["validation"] = validation
-
-        validation_status = validation.get("status")
-        if validation_status == "fail":
-            fail()
-            error_count = validation.get("summary", {}).get("errors", 0)
-            reasons.append(f"Contract validation failed with {error_count} error(s).")
-        elif validation_status == "warn":
-            warning_count = validation.get("summary", {}).get("warnings", 0)
-            if fail_on_validation_warning:
-                fail()
-                reasons.append(
-                    f"Contract validation produced {warning_count} warning(s), promoted to failure."
-                )
-            else:
-                warn()
-                reasons.append(f"Contract validation produced {warning_count} warning(s).")
-
-    if reference is not None:
-        reference_df, reference_name = _load_input(reference, label="Reference")
-        drift_payload = compare_datasets(
-            reference_df,
-            current_df,
-            columns=columns,
-            max_columns=max_columns,
-        )
-        drift_payload["reference_name"] = reference_name
-        drift_payload["current_name"] = current_name
-        checks["drift"] = drift_payload
-
-        if not drift_payload.get("available"):
-            fail()
-            reasons.append(
-                "Drift comparison was requested but could not produce a comparable result: "
-                f"{drift_payload.get('reason', 'unknown reason')}."
-            )
-        else:
-            drift_severity = str(
-                drift_payload.get("gate", {}).get("severity", "unknown")
-            )
-            if drift_severity not in _DRIFT_SEVERITY_RANK:
-                fail()
-                reasons.append(
-                    "Drift comparison did not produce a recognized severity verdict."
-                )
-            elif severity_at_least(drift_severity, drift_fail_on):
-                fail()
-                reasons.append(
-                    f"Drift severity {drift_severity} reached fail threshold {drift_fail_on}."
-                )
-            elif severity_at_least(drift_severity, drift_warn_on):
-                warn()
-                reasons.append(
-                    f"Drift severity {drift_severity} reached warning threshold {drift_warn_on}."
-                )
-
-            drift_reasons = drift_payload.get("gate", {}).get("reasons", [])
-            if isinstance(drift_reasons, list):
-                for reason in drift_reasons[:10]:
-                    text = str(reason)
-                    if text and text not in reasons:
-                        reasons.append(text)
-
-    return GateResult({
-        "status": status,
-        "passed": status != "fail",
-        "current_name": current_name,
-        "checks_run": list(checks),
-        "thresholds": {
-            "drift_warn_on": drift_warn_on,
-            "drift_fail_on": drift_fail_on,
-            "fail_on_validation_warning": bool(fail_on_validation_warning),
-        },
-        "reasons": reasons,
-        "checks": checks,
-    })
+    """Run the canonical CI-friendly contract/drift quality gate."""
+    from framevitals.operations import gate as _gate
+
+    return _gate(
+        current,
+        reference=reference,
+        contract=contract,
+        columns=columns,
+        max_columns=max_columns,
+        drift_warn_on=drift_warn_on,
+        drift_fail_on=drift_fail_on,
+        fail_on_validation_warning=fail_on_validation_warning,
+    )
+
+
+__all__ = [
+    "profile",
+    "roles",
+    "health",
+    "ml_readiness",
+    "quality",
+    "statistics",
+    "anomalies",
+    "relationships",
+    "target_analysis",
+    "analyze",
+    "plan",
+    "plan_cleaning",
+    "clean",
+    "compare",
+    "infer_contract",
+    "validate",
+    "gate",
+]

From 096d038db727f6bd8b86727352b7593a59179c65 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:27:12 +0530
Subject: [PATCH 222/439] test: lock canonical API facade boundaries

---
 tests/test_api_facade.py | 70 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 tests/test_api_facade.py

diff --git a/tests/test_api_facade.py b/tests/test_api_facade.py
new file mode 100644
index 0000000..404eef1
--- /dev/null
+++ b/tests/test_api_facade.py
@@ -0,0 +1,70 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def test_compat_api_import_does_not_eagerly_load_data_stack():
+    src_root = Path(__file__).resolve().parents[1] / "src"
+    env = os.environ.copy()
+    env["PYTHONPATH"] = str(src_root)
+    completed = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import sys; import framevitals.api; "
+                "assert 'pandas' not in sys.modules; "
+                "assert 'framevitals.pipeline' not in sys.modules; "
+                "assert 'framevitals.anomaly_ensemble' not in sys.modules"
+            ),
+        ],
+        cwd=src_root.parent,
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert completed.returncode == 0, completed.stderr
+
+
+def test_compat_api_delegates_compare_to_operations(monkeypatch):
+    import framevitals.api as api
+    import framevitals.operations as operations
+
+    expected = {"available": True, "marker": "canonical-compare"}
+
+    def fake_compare(reference, current, *, columns=None, max_columns=30):
+        assert reference == "reference"
+        assert current == "current"
+        assert columns == ["value"]
+        assert max_columns == 4
+        return expected
+
+    monkeypatch.setattr(operations, "compare", fake_compare)
+    result = api.compare(
+        "reference",
+        "current",
+        columns=["value"],
+        max_columns=4,
+    )
+
+    assert result is expected
+
+
+def test_compat_api_delegates_analyze_to_source_dispatcher(monkeypatch):
+    import framevitals.analysis_api as analysis_api
+    import framevitals.api as api
+
+    expected = {"marker": "canonical-analysis"}
+
+    def fake_analyze(data, **kwargs):
+        assert data == "dataset.csv"
+        assert kwargs["mode"] == "quick"
+        assert kwargs["artifacts"] is False
+        return expected
+
+    monkeypatch.setattr(analysis_api, "analyze", fake_analyze)
+    result = api.analyze("dataset.csv", mode="quick", artifacts=False)
+
+    assert result is expected

From 928646b5003d3814718552dda7cb758c32b6a2c8 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:27:53 +0530
Subject: [PATCH 223/439] feat: add first-class quality gate CLI

---
 src/framevitals/cli.py | 108 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 99 insertions(+), 9 deletions(-)

diff --git a/src/framevitals/cli.py b/src/framevitals/cli.py
index a978c72..b4d1cbb 100644
--- a/src/framevitals/cli.py
+++ b/src/framevitals/cli.py
@@ -71,6 +71,13 @@ def _load_contract(path: Path) -> dict:
     return payload
 
 
+def _parse_columns(value: str | None) -> list[str] | None:
+    if not value:
+        return None
+    columns = [item.strip() for item in value.split(",") if item.strip()]
+    return columns or None
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog="framevitals",
@@ -249,6 +256,59 @@ def build_parser() -> argparse.ArgumentParser:
     )
     _add_output_argument(validate_parser)
 
+    gate_parser = subparsers.add_parser(
+        "gate",
+        help="Run contract and/or drift checks as one CI-friendly quality gate.",
+    )
+    gate_parser.add_argument("current", type=Path, help="Current dataset path.")
+    gate_parser.add_argument(
+        "--reference",
+        type=Path,
+        default=None,
+        help="Optional reference dataset for drift checks.",
+    )
+    gate_parser.add_argument(
+        "--contract",
+        type=Path,
+        default=None,
+        help="Optional JSON data contract for exact validation.",
+    )
+    gate_parser.add_argument(
+        "--columns",
+        default=None,
+        help="Optional comma-separated columns to include in drift checks.",
+    )
+    gate_parser.add_argument(
+        "--max-columns",
+        type=int,
+        default=30,
+        help="Maximum number of shared columns to compare for drift.",
+    )
+    gate_parser.add_argument(
+        "--drift-warn-on",
+        choices=["stable", "minor", "moderate", "severe"],
+        default="moderate",
+        help="Drift severity that changes a passing gate to warning.",
+    )
+    gate_parser.add_argument(
+        "--drift-fail-on",
+        choices=["stable", "minor", "moderate", "severe"],
+        default="severe",
+        help="Drift severity that fails the gate.",
+    )
+    gate_parser.add_argument(
+        "--fail-on-validation-warning",
+        action="store_true",
+        help="Promote contract validation warnings to gate failures.",
+    )
+    gate_parser.add_argument(
+        "--format",
+        choices=["json", "terminal"],
+        default="terminal",
+        help="Stdout format.",
+    )
+    _add_output_argument(gate_parser)
+
     config_parser = subparsers.add_parser(
         "config",
         help="Resolve and inspect FrameVitals runtime configuration.",
@@ -333,6 +393,22 @@ def _render_compare(report: dict) -> str:
     return "\n".join(lines)
 
 
+def _render_gate(report: dict) -> str:
+    lines = [
+        "FrameVitals gate",
+        f"Status          {str(report.get('status', 'unknown')).upper()}",
+        f"Passed          {'yes' if report.get('passed') else 'no'}",
+        f"Checks          {', '.join(report.get('checks_run', [])) or 'none'}",
+    ]
+    reasons = report.get("reasons", [])
+    if reasons:
+        lines.append("")
+        lines.append("Reasons")
+        for reason in reasons[:10]:
+            lines.append(f"- {reason}")
+    return "\n".join(lines)
+
+
 def main() -> int:
     parser = build_parser()
     args = parser.parse_args()
@@ -430,18 +506,10 @@ def main() -> int:
         from framevitals.drift_analysis import severity_at_least
         from framevitals.operations import compare
 
-        columns = None
-        if args.columns:
-            columns = [
-                value.strip()
-                for value in args.columns.split(",")
-                if value.strip()
-            ]
-
         report = compare(
             args.reference,
             args.current,
-            columns=columns,
+            columns=_parse_columns(args.columns),
             max_columns=args.max_columns,
         )
         if args.output is not None:
@@ -493,6 +561,28 @@ def main() -> int:
             return 1
         return 0
 
+    if args.command == "gate":
+        from framevitals.operations import gate
+
+        contract = _load_contract(args.contract) if args.contract is not None else None
+        report = gate(
+            args.current,
+            reference=args.reference,
+            contract=contract,
+            columns=_parse_columns(args.columns),
+            max_columns=args.max_columns,
+            drift_warn_on=args.drift_warn_on,
+            drift_fail_on=args.drift_fail_on,
+            fail_on_validation_warning=args.fail_on_validation_warning,
+        )
+        if args.output is not None:
+            _write_json(report, args.output)
+        if args.format == "json":
+            print(json.dumps(report, indent=2, default=str))
+        else:
+            print(_render_gate(report))
+        return 0 if report.get("passed") else 1
+
     if args.command == "config":
         from framevitals.config import resolve_config
 

From be72cec6c939738db3c8e97f944c411a9fd444eb Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:29:08 +0530
Subject: [PATCH 224/439] test: cover quality gate CLI semantics

---
 tests/test_cli_gate.py | 123 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 123 insertions(+)
 create mode 100644 tests/test_cli_gate.py

diff --git a/tests/test_cli_gate.py b/tests/test_cli_gate.py
new file mode 100644
index 0000000..b801bde
--- /dev/null
+++ b/tests/test_cli_gate.py
@@ -0,0 +1,123 @@
+import json
+
+from framevitals.cli import build_parser, main
+
+
+def test_gate_parser_exposes_ci_controls():
+    parser = build_parser()
+    args = parser.parse_args([
+        "gate",
+        "production.csv",
+        "--reference",
+        "training.csv",
+        "--contract",
+        "contract.json",
+        "--columns",
+        "age,income",
+        "--max-columns",
+        "12",
+        "--drift-warn-on",
+        "minor",
+        "--drift-fail-on",
+        "moderate",
+        "--fail-on-validation-warning",
+        "--format",
+        "json",
+        "--output",
+        "gate.json",
+    ])
+
+    assert args.command == "gate"
+    assert args.current.name == "production.csv"
+    assert args.reference.name == "training.csv"
+    assert args.contract.name == "contract.json"
+    assert args.columns == "age,income"
+    assert args.max_columns == 12
+    assert args.drift_warn_on == "minor"
+    assert args.drift_fail_on == "moderate"
+    assert args.fail_on_validation_warning is True
+    assert args.format == "json"
+    assert args.output.name == "gate.json"
+
+
+def test_gate_cli_returns_zero_for_passing_contract(tmp_path, monkeypatch, capsys):
+    dataset = tmp_path / "candidate.csv"
+    contract = tmp_path / "contract.json"
+    dataset.write_text("age,plan\n25,basic\n30,pro\n", encoding="utf-8")
+    contract.write_text(
+        json.dumps({
+            "version": 2,
+            "allow_extra_columns": False,
+            "columns": {
+                "age": {
+                    "type": "integer",
+                    "nullable": False,
+                    "minimum": 18,
+                    "maximum": 100,
+                },
+                "plan": {
+                    "type": "string",
+                    "nullable": False,
+                    "allowed_values": ["basic", "pro"],
+                },
+            },
+        }),
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "gate",
+            str(dataset),
+            "--contract",
+            str(contract),
+            "--format",
+            "json",
+        ],
+    )
+
+    assert main() == 0
+    payload = json.loads(capsys.readouterr().out)
+    assert payload["status"] == "pass"
+    assert payload["passed"] is True
+    assert payload["checks_run"] == ["validation"]
+
+
+def test_gate_cli_returns_one_for_failing_contract(tmp_path, monkeypatch, capsys):
+    dataset = tmp_path / "candidate.csv"
+    contract = tmp_path / "contract.json"
+    dataset.write_text("age\n15\n", encoding="utf-8")
+    contract.write_text(
+        json.dumps({
+            "version": 2,
+            "allow_extra_columns": False,
+            "columns": {
+                "age": {
+                    "type": "integer",
+                    "nullable": False,
+                    "minimum": 18,
+                    "maximum": 100,
+                },
+            },
+        }),
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "gate",
+            str(dataset),
+            "--contract",
+            str(contract),
+            "--format",
+            "json",
+        ],
+    )
+
+    assert main() == 1
+    payload = json.loads(capsys.readouterr().out)
+    assert payload["status"] == "fail"
+    assert payload["passed"] is False
+    assert payload["checks"]["validation"]["status"] == "fail"

From 65e625c515a393f413f681e083c6b03630b5071e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:29:17 +0530
Subject: [PATCH 225/439] test: prove CLI gate preserves Parquet streaming

---
 tests/test_cli_gate_streaming.py | 58 ++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 tests/test_cli_gate_streaming.py

diff --git a/tests/test_cli_gate_streaming.py b/tests/test_cli_gate_streaming.py
new file mode 100644
index 0000000..83e3b2c
--- /dev/null
+++ b/tests/test_cli_gate_streaming.py
@@ -0,0 +1,58 @@
+import json
+
+import numpy as np
+import pandas as pd
+import pytest
+
+pa = pytest.importorskip("pyarrow")
+pq = pytest.importorskip("pyarrow.parquet")
+
+from framevitals.cli import main
+from framevitals.sources import ParquetSource
+
+
+def _write_parquet(path, *, rows: int, shift: float = 0.0) -> None:
+    frame = pd.DataFrame({
+        "value": np.arange(rows, dtype=np.float64) + shift,
+        "group": [f"g-{index % 5}" for index in range(rows)],
+    })
+    pq.write_table(
+        pa.Table.from_pandas(frame, preserve_index=False),
+        path,
+        row_group_size=777,
+    )
+
+
+def test_cli_reference_gate_streams_parquet(tmp_path, monkeypatch, capsys):
+    reference = tmp_path / "reference.parquet"
+    current = tmp_path / "current.parquet"
+    _write_parquet(reference, rows=60_000)
+    _write_parquet(current, rows=72_000, shift=50.0)
+
+    def fail_load(self):
+        raise AssertionError("CLI reference gate must not fully materialize Parquet inputs")
+
+    monkeypatch.setattr(ParquetSource, "load", fail_load)
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "gate",
+            str(current),
+            "--reference",
+            str(reference),
+            "--format",
+            "json",
+        ],
+    )
+
+    exit_code = main()
+    assert exit_code in {0, 1}
+    payload = json.loads(capsys.readouterr().out)
+    execution = payload["execution"]["drift"]
+    assert execution["full_materialization"] is False
+    assert execution["reference"]["source_rows"] == 60_000
+    assert execution["reference"]["sample_rows"] == 50_000
+    assert execution["current"]["source_rows"] == 72_000
+    assert execution["current"]["sample_rows"] == 50_000
+    assert execution["components"]["value_distributions"] == "bounded_row_sample"

From 728782680b4b9e51bb0131224f945bf1df19fa91 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:29:42 +0530
Subject: [PATCH 226/439] ci: exercise streaming quality gate CLI

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0996dda..2d9888f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -230,7 +230,7 @@ jobs:
       - name: Run streaming source tests without native extension
         env:
           FRAMEVITALS_BACKEND: numpy
-        run: pytest -v tests/test_parquet_streaming.py tests/test_statistics_streaming.py tests/test_streaming_drift.py tests/test_csv_streaming.py
+        run: pytest -v tests/test_parquet_streaming.py tests/test_statistics_streaming.py tests/test_streaming_drift.py tests/test_csv_streaming.py tests/test_cli_gate_streaming.py
 
   optional-features:
     name: optional features (Python 3.12)

From d17a3a0e3b2134eac6683453697a2e6f6a9fa905 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:30:40 +0530
Subject: [PATCH 227/439] docs: align README with source-aware quality gate
 architecture

---
 README.md | 331 ++++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 246 insertions(+), 85 deletions(-)

diff --git a/README.md b/README.md
index 270765e..964518d 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@
 
 ### Know if your data is healthy, stable, and ML-ready — before your model finds out.
 
-**A Python toolkit for data-quality diagnostics, drift detection, anomaly analysis, and ML-readiness checks on pandas and tabular data.**
+**A Python toolkit for data-health diagnostics, drift detection, contracts, quality gates, anomaly analysis, and ML-readiness checks on tabular data.**
 
 [![Tests](https://github.com/parthdongre/FrameVitals/actions/workflows/test.yml/badge.svg)](https://github.com/parthdongre/FrameVitals/actions/workflows/test.yml)
 [![PyPI](https://img.shields.io/pypi/v/framevitals.svg)](https://pypi.org/project/framevitals/)
@@ -12,54 +12,65 @@
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
 [![GitHub stars](https://img.shields.io/github/stars/parthdongre/FrameVitals?style=social)](https://github.com/parthdongre/FrameVitals)
 
-[Install](#installation) · [Quick start](#quick-start) · [CLI](#command-line-interface) · [Roadmap](#roadmap) · [Contributing](#contributing)
+[Install](#installation) · [Quick start](#quick-start) · [Quality gates](#quality-gates) · [CLI](#command-line-interface) · [Roadmap](#roadmap) · [Contributing](#contributing)
 
 </div>
 
 ---
 
-FrameVitals turns a pandas DataFrame or tabular dataset into a **structured health report** you can inspect, serialize, compare, and eventually enforce in CI.
+FrameVitals turns a pandas DataFrame or tabular dataset into a **structured health report** that can be inspected, serialized, compared, snapshotted, and enforced in CI.
 
-Instead of stitching together separate profiling, quality, drift, anomaly, and ML-readiness tools, FrameVitals gives you one deliberately small entry point:
+Instead of stitching together separate profiling, quality, drift, anomaly, contract, and ML-readiness tools, FrameVitals exposes a compact workflow:
 
 ```python
 import framevitals as fv
 
-report = fv.analyze(df)
-drift = fv.compare(reference_df, current_df)
-contract = fv.infer_contract(reference_df)
-validation = fv.validate(current_df, contract)
+report = fv.analyze(data)
+drift = fv.compare(reference, current)
+contract = fv.infer_contract(reference)
+validation = fv.validate(current, contract)
+gate = fv.gate(current, reference=reference, contract=contract)
 ```
 
 The goal is simple: **catch bad data before it becomes a bad model, a broken dashboard, or a production incident.**
 
 ```text
-                  ┌──────────────────────────┐
-DataFrame / file ─►        ANALYZE           │
-                  │ profile · health · ML    │
-                  │ stats · anomalies · risk │
-                  └────────────┬─────────────┘
-                               │
-                               ▼
-                        structured report
-
-Reference + current ───────────► COMPARE ─────► drift verdict
+                         ┌──────────────────────────┐
+DataFrame / file ───────►│         ANALYZE          │
+                         │ profile · health · ML     │
+                         │ stats · anomalies · risk  │
+                         └────────────┬─────────────┘
+                                      │
+                                      ▼
+                               structured result
+                                      │
+                        ┌─────────────┴─────────────┐
+                        ▼                           ▼
+                    snapshot                    contract
+                        │                           │
+Reference + current ────┴──► compare / validate ───┘
+                                      │
+                                      ▼
+                                    GATE
+                              PASS / WARN / FAIL
 ```
 
 ## Why FrameVitals?
 
-Most data checks answer one narrow question. FrameVitals is designed around the questions that show up repeatedly in real data and ML workflows:
+Most data checks answer one narrow question. FrameVitals is designed around the questions that repeatedly show up in real data and ML workflows:
 
 | Question | FrameVitals |
 | --- | --- |
 | Is this dataset structurally healthy? | Missingness, duplicates, cardinality, schema and quality diagnostics |
 | Is it ready for modelling? | ML-readiness scoring, target-aware checks and model diagnostics |
 | Are there suspicious rows or features? | Statistical diagnostics, anomaly detection, leakage and multicollinearity checks |
-| Has production data changed? | Reference-vs-current drift analysis with numeric and categorical tests |
-| Can I use the result in code? | JSON-friendly structured output through a Python API and CLI |
+| Has production data changed? | Reference-vs-current numeric, categorical and schema drift diagnostics |
+| Can I enforce expectations? | Versioned data contracts, validation and one CI-friendly quality gate |
+| Can I track change without storing every raw dataset? | Compact versioned analysis snapshots and snapshot diffs |
+| Will large Arrow-compatible files be fully loaded by default? | Source-aware bounded execution for supported streaming workflows |
 | Will analysis unexpectedly write files? | No — filesystem artifacts are opt-in |
 
-FrameVitals is **package-first**. The core library lives under `src/framevitals/`; the Flask API and React dashboard are optional interfaces around the same analysis engine.
+FrameVitals is **package-first**. The canonical library lives under `src/framevitals/`; the Flask API and React dashboard are optional interfaces around the same analysis engine.
 
 ## Installation
 
@@ -69,15 +80,18 @@ FrameVitals supports **Python 3.11, 3.12, and 3.13**. The current public release
 pip install framevitals
 ```
 
-Optional feature groups keep heavier dependencies out of the default install:
+Optional capabilities are installed as extras:
 
 ```bash
-pip install "framevitals[ml]"   # XGBoost, LightGBM, PyOD, SHAP
-pip install "framevitals[ai]"   # Ollama-backed AI features
-pip install "framevitals[web]"  # Flask web runtime
-pip install "framevitals[all]"  # all optional runtime features
+pip install "framevitals[arrow]"  # bounded Parquet / compatible CSV / TSV streaming
+pip install "framevitals[ml]"     # XGBoost, LightGBM, PyOD, SHAP
+pip install "framevitals[ai]"     # Ollama-backed AI features
+pip install "framevitals[web]"    # Flask web runtime
+pip install "framevitals[all]"    # all optional runtime capabilities
 ```
 
+Parquet streaming requires the `arrow` extra. With Arrow installed, compatible CSV and TSV inputs can also use the source-aware streaming engine; without Arrow they retain the normal pandas-backed file path.
+
 ## Quick start
 
 ### Analyze a DataFrame
@@ -89,8 +103,9 @@ import framevitals as fv
 customers = pd.read_csv("customers.csv")
 report = fv.analyze(customers)
 
-print(report["health"]["overall_score"])
-print(report["ml_readiness"])
+print(report.health["overall_score"])
+print(report.ml_readiness)
+print(report.findings[:3])
 ```
 
 File paths work too:
@@ -99,7 +114,22 @@ File paths work too:
 report = fv.analyze("customers.csv", mode="quick")
 ```
 
-FrameVitals supports pandas DataFrames and common tabular file formats including CSV, TSV, Excel, and JSON.
+FrameVitals supports pandas DataFrames and common tabular file formats including CSV, TSV, Excel, JSON, and Parquet when the required reader capability is installed.
+
+### Run only the diagnostic you need
+
+The focused API avoids running the complete analysis pipeline when only one answer is required:
+
+```python
+profile = fv.profile("customers.parquet")
+health = fv.health("customers.parquet")
+readiness = fv.ml_readiness("customers.parquet")
+stats = fv.statistics("customers.parquet", mode="quick")
+anomalies = fv.anomalies("customers.parquet", mode="quick")
+relationships = fv.relationships("customers.parquet")
+```
+
+Supported streaming sources use bounded execution where the diagnostic permits it and disclose execution provenance in their result metadata.
 
 ### Add a supervised-learning target
 
@@ -128,39 +158,100 @@ print(result["summary"]["overall_verdict"])
 print(result["columns"][:3])
 ```
 
-Numeric drift uses **PSI, Kolmogorov-Smirnov statistics, and standardized mean shift**. Categorical drift uses **PSI and chi-square diagnostics**.
+Numeric drift uses PSI, Kolmogorov-Smirnov statistics, and standardized mean shift. Categorical drift uses PSI and chi-square diagnostics.
+
+For streaming-capable file sources, value-distribution work is bounded and the result reports the true source shapes plus the sampling strategy used for distribution diagnostics.
 
 ### Validate a data contract
 
-Infer a contract once from a trusted reference dataset, then validate later
-batches before they reach downstream jobs:
+Infer a contract once from a trusted reference dataset, then validate later batches before they reach downstream jobs:
 
 ```python
 contract = fv.infer_contract(reference)
 result = fv.validate(current, contract)
 
-if not result["valid"]:
+if result["status"] == "fail":
     for finding in result["errors"]:
         print(finding["message"])
 ```
 
-Contracts capture required columns, broad data types, nullability, and finite
-numeric bounds. They are plain JSON-friendly dictionaries, so a contract can
-be committed with a pipeline or stored with a dataset baseline.
+Contracts are versioned JSON-friendly dictionaries. Current contract inference can capture required or optional columns, broad data types, nullability and tolerated null rates, finite numeric bounds, low-cardinality allowed values, and uniqueness expectations.
 
-## The public API
+Validation intentionally remains exact for constraints such as uniqueness, allowed values and bounds rather than silently downgrading those checks to sampled approximations.
+
+## Quality gates
+
+`fv.gate(...)` combines contract validation and drift into one CI-friendly verdict:
+
+```python
+result = fv.gate(
+    current,
+    reference=reference,
+    contract=contract,
+    drift_warn_on="moderate",
+    drift_fail_on="severe",
+)
+
+print(result.status)   # pass / warn / fail
+print(result.passed)   # True unless the gate failed
+```
+
+You can run only the checks you need:
+
+```python
+fv.gate(current, contract=contract)
+fv.gate(current, reference=reference)
+fv.gate(current, reference=reference, contract=contract)
+```
+
+Contract validation remains exact. Drift can use bounded source-aware sampling where a streaming input supports it, and the returned `execution` block records that distinction.
 
-The public API is intentionally small while FrameVitals is in alpha.
+## Snapshots
 
-| API | Status | Purpose |
-| --- | --- | --- |
-| `framevitals.analyze(...)` | Available in `0.1.0` | Profile and diagnose one dataset |
-| `framevitals.compare(...)` | Available in `0.1.0` | Compare reference and current data for drift |
-| `framevitals.infer_contract(...)` | Available on `dev` | Infer a reusable data contract from reference data |
-| `framevitals.validate(...)` | Available on `dev` | Validate data against an inferred or explicit contract |
-| snapshots / monitoring | Roadmap | Reuse baselines for recurring schema and drift checks |
+An `AnalysisResult` can be reduced to a compact versioned monitoring snapshot:
 
-This keeps the library easy to learn while leaving room for the result model and validation system to mature before `1.0`.
+```python
+report = fv.analyze(current)
+snapshot = report.snapshot("snapshot.json")
+```
+
+Snapshots retain a fingerprint plus compact state such as schema, missingness, health, ML-readiness and finding codes without embedding the full raw dataset.
+
+```python
+previous = fv.load_snapshot("previous.json")
+latest = fv.load_snapshot("snapshot.json")
+change = fv.compare_snapshots(previous, latest)
+
+print(change["schema"])
+print(change["health_delta"])
+print(change["findings"]["new"])
+```
+
+## The public API
+
+FrameVitals keeps workflow entry points at the package root and exposes focused diagnostics for callers that do not need the full pipeline.
+
+| API | Purpose |
+| --- | --- |
+| `framevitals.analyze(...)` | Run the configured source-aware analysis workflow |
+| `framevitals.plan(...)` | Preview applicable modules and execution constraints |
+| `framevitals.profile(...)` | Profile structure, missingness and summaries only |
+| `framevitals.roles(...)` | Infer semantic and structural column roles |
+| `framevitals.health(...)` | Calculate data-health diagnostics only |
+| `framevitals.ml_readiness(...)` | Calculate ML-readiness diagnostics only |
+| `framevitals.quality(...)` | Run deterministic data-quality checks only |
+| `framevitals.statistics(...)` | Run bounded deep statistical diagnostics |
+| `framevitals.anomalies(...)` | Run bounded anomaly diagnostics |
+| `framevitals.relationships(...)` | Discover strong numeric relationships |
+| `framevitals.target_analysis(...)` | Run target-aware diagnostics only |
+| `framevitals.compare(...)` | Compare reference/current data for drift |
+| `framevitals.infer_contract(...)` | Infer a reusable versioned data contract |
+| `framevitals.validate(...)` | Validate data against a contract |
+| `framevitals.gate(...)` | Combine validation and drift into one quality verdict |
+| `framevitals.create_snapshot(...)` | Create compact monitoring state from an analysis result |
+| `framevitals.compare_snapshots(...)` | Compare two stored analysis states |
+
+The compatibility module `framevitals.api` delegates lazily to these same canonical engines so older imports do not maintain a second implementation.
 
 ## What FrameVitals checks
 
@@ -174,19 +265,20 @@ This keeps the library easy to learn while leaving room for the result model and
 | **Anomalies** | multivariate and robust outlier detectors, optional ensemble methods |
 | **Target intelligence** | task inference, leakage hints, multicollinearity, feature/model diagnostics |
 | **Drift** | PSI, KS, chi-square, mean shift, new or disappearing categories |
+| **Contracts** | schema, types, nullability, bounds, domains and uniqueness expectations |
 | **Time series** | date-aware diagnostics, stationarity, decomposition and forecast previews |
 | **Text** | text-column profiling, vocabulary and lightweight semantic diagnostics |
 | **Explainability** | model feature importance and SHAP when the optional ML stack is installed |
 
-Not every analysis runs on every dataset. FrameVitals uses dataset signals, selected mode, target availability, and installed optional dependencies to decide what is useful and safe to execute.
+Not every analysis runs on every dataset. FrameVitals uses dataset signals, selected mode, target availability, source capabilities, execution budgets, and installed optional dependencies to decide what is useful and safe to execute.
 
 ## Analysis modes
 
 ```python
-fv.analyze(df, mode="quick")
-fv.analyze(df, mode="standard")
-fv.analyze(df, mode="deep")
-fv.analyze(df, mode="research")
+fv.analyze(data, mode="quick")
+fv.analyze(data, mode="standard")
+fv.analyze(data, mode="deep")
+fv.analyze(data, mode="research")
 ```
 
 | Mode | Best for |
@@ -196,28 +288,50 @@ fv.analyze(df, mode="research")
 | `deep` | Target-aware and heavier statistical analysis |
 | `research` | Largest analysis budget for exploratory work |
 
+## Source-aware execution
+
+FrameVitals separates a dataset source from the diagnostic that consumes it. That lets supported file formats expose metadata, projection and record batches without forcing every public API to begin with a full pandas materialization.
+
+```text
+DataFrame / CSV / TSV / Parquet / ...
+                  │
+                  ▼
+            DatasetSource
+                  │
+        ┌─────────┴─────────┐
+        ▼                   ▼
+   streaming path      materialized path
+        │                   │
+        └─────────┬─────────┘
+                  ▼
+         bounded diagnostics
+```
+
+Parquet is projection-aware and streaming through the Arrow capability. Compatible CSV/TSV sources can also use Arrow record batches. Focused diagnostics disclose when a result is exact, derived from a bounded row sample, or required full materialization.
+
 ## Filesystem artifacts are opt-in
 
-FrameVitals is designed to behave like a library first. Calling the Python API does not need to scatter reports and cleaned files around your working directory.
+FrameVitals is designed to behave like a library first. Calling the Python API does not need to scatter reports and cleaned files around the working directory.
 
 ```python
-report = fv.analyze(df)
+report = fv.analyze(data)
 assert report["cleaning"]["output_path"] is None
 
-report = fv.analyze(df, artifacts=True)
+report = fv.analyze(data, artifacts=True)
 print(report["cleaning"]["output_path"])
 ```
 
 ## Command-line interface
 
-FrameVitals also ships with a CLI for scripts, terminals, and future CI workflows.
+FrameVitals ships with a CLI for scripts, terminals and CI workflows.
 
-Start by discovering the available commands and options:
+Discover commands and options:
 
 ```bash
 framevitals --help
 framevitals analyze --help
 framevitals compare --help
+framevitals gate --help
 framevitals --version
 ```
 
@@ -231,10 +345,10 @@ framevitals analyze dataset.csv --output report.json
 framevitals analyze dataset.csv --artifacts
 ```
 
-A useful end-to-end smoke test is:
+Preview work without executing the heavy stages:
 
 ```bash
-framevitals analyze dataset.csv --target churn --mode deep --artifacts --output report.json
+framevitals plan dataset.csv --mode standard
 ```
 
 Compare two datasets for drift:
@@ -242,30 +356,72 @@ Compare two datasets for drift:
 ```bash
 framevitals compare train.csv production.csv
 framevitals compare train.csv production.csv --columns age,income
+framevitals compare train.csv production.csv --fail-on moderate
 framevitals compare train.csv production.csv --output drift.json
 ```
 
-Create and use a contract from the terminal:
+Create and validate a contract:
 
 ```bash
 framevitals infer-contract training_data.csv --output contract.json
 framevitals validate production_batch.csv --contract contract.json
 ```
 
-The validation command exits with status `1` when the contract has errors,
-which makes it suitable for CI jobs and scheduled ingestion checks.
+Run the combined quality gate:
+
+```bash
+framevitals gate production_batch.csv \
+  --reference training_data.csv \
+  --contract contract.json
+```
+
+Tune CI thresholds when needed:
+
+```bash
+framevitals gate production_batch.csv \
+  --reference training_data.csv \
+  --contract contract.json \
+  --drift-warn-on moderate \
+  --drift-fail-on severe \
+  --fail-on-validation-warning \
+  --output gate.json
+```
+
+CLI exit behavior is intentional:
+
+- `framevitals validate`: `0` for pass/warn by default, `1` for a warning when `--fail-on-warn` is enabled, and `2` for contract failure.
+- `framevitals compare`: `1` only when `--fail-on` is supplied and the configured drift severity is reached; otherwise `0`.
+- `framevitals gate`: `0` for pass/warn and `1` for fail.
+
+## Result objects
+
+`fv.analyze(...)` returns an `AnalysisResult`, a dict-compatible result object with notebook- and application-friendly helpers:
+
+```python
+report = fv.analyze(data)
+
+report.health
+report.ml_readiness
+report.findings
+report.recommendations
+report.column("age")
+report.summary()
+report.to_json("report.json")
+report.to_html("report.html")
+report.snapshot("snapshot.json")
+```
 
-In `0.1.0`, `framevitals analyze` prints a compact analysis summary to the terminal. `--output report.json` writes that CLI summary to the requested path. `--artifacts` enables generated files in the current working directory, including a cleaned dataset under `cleaned/` and generated charts under `static/charts/`. The full structured analysis result is available through the Python API with `framevitals.analyze(...)`.
+Drift, validation and gate workflows similarly return dict-compatible `DriftResult`, `ValidationResult` and `GateResult` objects with convenience properties while preserving JSON-friendly mapping behavior during the `0.x` series.
 
 ## Optional ML and AI features
 
-The default package contains the core data-health engine. Heavier features are separated into extras so a simple install stays predictable.
+The default package contains the core data-health engine. Additional integrations are separated into extras.
 
 ```bash
 pip install "framevitals[ml]"
 ```
 
-Adds optional integrations including XGBoost, LightGBM, PyOD and SHAP.
+Adds integrations including XGBoost, LightGBM, PyOD and SHAP.
 
 ```bash
 pip install "framevitals[ai]"
@@ -295,18 +451,21 @@ Typical local endpoints:
 - Flask API: `http://127.0.0.1:5055`
 - React dashboard: `http://127.0.0.1:5173`
 
-The public project website will remain separate from the package runtime so the library does not depend on a hosted service.
+The public project website can remain separate from the package runtime so the Python library does not depend on a hosted service.
 
 ## Design principles
 
 FrameVitals is being built around a few constraints that are easy to lose in analytics projects:
 
-- **DataFrame first** — use it directly from Python without routing through a web app.
+- **Package first** — the Python API is canonical; web interfaces wrap it rather than define it.
 - **Structured results** — return reusable data, not only screenshots or prose.
+- **Source aware** — inspect and stream supported sources instead of eagerly loading everything by default.
+- **Bounded execution** — expensive diagnostics operate within explicit scale budgets.
+- **Transparent approximations** — sampled or approximate results disclose their execution provenance.
+- **Exact where correctness requires it** — contract constraints are not silently weakened to sampled checks.
 - **Safe defaults** — no unexpected artifact writes and graceful optional-feature fallbacks.
-- **Small public API** — make the common path obvious before exposing every internal module.
-- **Optional heavy dependencies** — ML, AI, and web features should not bloat a basic install.
-- **Production direction** — drift, contracts, snapshots, and CI quality gates are first-class roadmap items.
+- **Small workflow surface** — make analyze → compare → validate → gate obvious while keeping focused diagnostics available.
+- **Optional integrations** — ML, AI, Arrow and web capabilities can be installed independently.
 
 ## Project layout
 
@@ -314,6 +473,8 @@ FrameVitals is being built around a few constraints that are easy to lose in ana
 .
 ├── src/framevitals/          # canonical installable Python package
 ├── tests/                    # automated test suite
+├── benchmarks/               # scale/performance harnesses
+├── rust/                     # optional native acceleration core
 ├── frontend/                 # optional React + TypeScript dashboard
 ├── templates/                # Flask report pages
 ├── static/                   # web/report assets
@@ -347,44 +508,44 @@ On Windows PowerShell:
 .venv\Scripts\Activate.ps1
 ```
 
-CI validates the core package across Python 3.11–3.13, optional features, the React build, wheel contents, distribution metadata, and a clean-wheel install.
+CI validates the core package across Python 3.11–3.13, Arrow streaming paths, optional features, the Rust workspace/native bridge, React build, wheel contents, distribution metadata, and a clean-wheel install.
 
 Development is integrated through `dev`; `main` is kept release-ready.
 
 ## Roadmap
 
-FrameVitals is moving toward a complete data-health quality gate:
+FrameVitals is moving toward a dependable data-health quality gate rather than an ever-growing collection of unrelated analytics modules.
 
 ```text
-0.1  ANALYZE + COMPARE
-     data health · ML readiness · target diagnostics · drift
+0.1  FOUNDATION
+     analyze · focused diagnostics · drift · package/CLI baseline
 
-0.2  VALIDATE + SNAPSHOTS
-     data contracts · CI gates · reusable baselines
+0.2  SOURCE-AWARE QUALITY GATES
+     bounded streaming · contracts · validate · gate · snapshots · execution provenance
 
-0.3  RESULT OBJECTS + ADVANCED DRIFT
-     stronger result model · large-data handling · richer monitoring
+0.3  STABLE RESULTS + PERFORMANCE
+     result-schema hardening · performance regression CI · lighter installs · richer history
 
-0.4  EXTENSIBILITY + INTEGRATIONS
-     configurable checks · adapters · monitoring workflows
+0.4  EXTENSIBILITY + DATA ADAPTERS
+     custom checks · plugin surface · Arrow/Polars/DuckDB-style adapters · integrations
 
 1.0  STABLE DATA-HEALTH API
-     dependable analyze → compare → validate → monitor workflow
+     dependable analyze → compare → validate → gate → monitor workflow
 ```
 
-Near-term work is tracked through issues and the `dev` branch.
+Several `0.2` capabilities are already under active development on the repository development branches. Before `1.0`, naming, thresholds, result schemas and extension points may still evolve.
 
 ## Project status
 
-FrameVitals `0.1.x` is **alpha software**. The core API is usable, but the project is intentionally still refining naming, result schemas, thresholds, and extension points before `1.0`.
+FrameVitals `0.1.x` is **alpha software**. The project already has a usable package API, CLI, contracts, gates, snapshots, source-aware execution, tests and release tooling, but the `0.x` series deliberately leaves room to refine result schemas, thresholds, dependency boundaries and extension points before stability guarantees begin.
 
-If you are using FrameVitals in a project, feedback about real datasets, false positives, missing diagnostics, performance, and API ergonomics is especially valuable.
+Feedback on real datasets, false positives, missing diagnostics, performance, source compatibility and API ergonomics is especially valuable.
 
 ## Contributing
 
 Contributions are welcome.
 
-A good contribution is focused, tested, and improves either the reliability of a diagnostic or the clarity of the public workflow.
+A good contribution is focused, tested, and improves either the reliability of a diagnostic, the clarity of the public workflow, source compatibility, or execution transparency.
 
 Start with [CONTRIBUTING.md](CONTRIBUTING.md), and please read the [Code of Conduct](CODE_OF_CONDUCT.md) and [Security Policy](SECURITY.md).
 

From 14dab9fa311b587794b45cd1ad1ee5648313c297 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:32:24 +0530
Subject: [PATCH 228/439] packaging: move Excel and AI-only deps out of core

---
 pyproject.toml | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 3420961..52e844b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -37,15 +37,16 @@ dependencies = [
     "numpy>=1.26",
     "matplotlib>=3.8",
     "seaborn>=0.13",
-    "openpyxl>=3.1",
-    "xlrd>=2.0",
     "scipy>=1.13",
     "statsmodels>=0.14",
     "scikit-learn>=1.5",
-    "pydantic>=2.7",
 ]
 
 [project.optional-dependencies]
+excel = [
+    "openpyxl>=3.1",
+    "xlrd>=2.0",
+]
 ml = [
     "xgboost>=2.0",
     "lightgbm>=4.3",
@@ -54,6 +55,7 @@ ml = [
 ]
 ai = [
     "ollama>=0.3",
+    "pydantic>=2.7",
 ]
 web = [
     "flask>=3.0",
@@ -64,11 +66,14 @@ arrow = [
     "pyarrow>=25.0,<26",
 ]
 all = [
+    "openpyxl>=3.1",
+    "xlrd>=2.0",
     "xgboost>=2.0",
     "lightgbm>=4.3",
     "pyod>=2.0",
     "shap>=0.45",
     "ollama>=0.3",
+    "pydantic>=2.7",
     "flask>=3.0",
     "gunicorn>=22.0",
     "werkzeug>=3.0",
@@ -81,6 +86,9 @@ dev = [
     "build>=1.2",
     "twine>=5.1",
     "ruff>=0.6",
+    "openpyxl>=3.1",
+    "xlrd>=2.0",
+    "pydantic>=2.7",
 ]
 
 [project.scripts]

From 7c6a4e6cc09f334218d2e57584a16523abd02def Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:32:38 +0530
Subject: [PATCH 229/439] feat: explain optional Excel reader requirement

---
 src/framevitals/loader.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/loader.py b/src/framevitals/loader.py
index a36ab2b..9e55b7f 100644
--- a/src/framevitals/loader.py
+++ b/src/framevitals/loader.py
@@ -84,6 +84,17 @@ def _read_csv_tolerant(
     )
 
 
+def _read_excel(file_path: str | Path) -> pd.DataFrame:
+    """Read Excel input and surface a FrameVitals-specific dependency hint."""
+    try:
+        return pd.read_excel(file_path)
+    except ImportError as exc:
+        raise ImportError(
+            "Excel input requires the optional FrameVitals Excel capability. "
+            'Install it with: pip install "framevitals[excel]"'
+        ) from exc
+
+
 def load_dataset(
     file_path: str | Path,
 ) -> pd.DataFrame:
@@ -91,7 +102,7 @@ def load_dataset(
     Load a supported tabular dataset.
 
     Supported formats:
-    CSV, TSV, XLSX, XLS, and JSON.
+    CSV, TSV, XLSX, XLS, and JSON. Excel readers are optional.
     """
 
     path = Path(file_path)
@@ -111,7 +122,7 @@ def load_dataset(
         ".xlsx",
         ".xls",
     }:
-        return pd.read_excel(path)
+        return _read_excel(path)
 
     if suffix == ".json":
         try:

From cb76fc3f236a1848aeaa24121fe327cb61b9bc75 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:32:55 +0530
Subject: [PATCH 230/439] test: lock optional dependency boundaries

---
 tests/test_optional_dependency_boundaries.py | 40 ++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 tests/test_optional_dependency_boundaries.py

diff --git a/tests/test_optional_dependency_boundaries.py b/tests/test_optional_dependency_boundaries.py
new file mode 100644
index 0000000..2e3093e
--- /dev/null
+++ b/tests/test_optional_dependency_boundaries.py
@@ -0,0 +1,40 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from framevitals.loader import load_dataset
+
+
+def test_core_import_does_not_eagerly_load_ai_dependency():
+    src_root = Path(__file__).resolve().parents[1] / "src"
+    env = os.environ.copy()
+    env["PYTHONPATH"] = str(src_root)
+    completed = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            "import sys; import framevitals; assert 'pydantic' not in sys.modules",
+        ],
+        cwd=src_root.parent,
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert completed.returncode == 0, completed.stderr
+
+
+def test_excel_loader_reports_framevitals_extra(monkeypatch, tmp_path):
+    path = tmp_path / "dataset.xlsx"
+    path.write_bytes(b"placeholder")
+
+    def missing_engine(*args, **kwargs):
+        raise ImportError("missing Excel engine")
+
+    monkeypatch.setattr("framevitals.loader.pd.read_excel", missing_engine)
+
+    with pytest.raises(ImportError, match=r"framevitals\[excel\]"):
+        load_dataset(path)

From 0ed0778dd3d370975b2bf3b7a44349a4690523e2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:33:31 +0530
Subject: [PATCH 231/439] packaging: mark FrameVitals as typed

---
 src/framevitals/py.typed | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 src/framevitals/py.typed

diff --git a/src/framevitals/py.typed b/src/framevitals/py.typed
new file mode 100644
index 0000000..e69de29

From 5e19aa27243e85d8ab779a666379a569f9b87bd3 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:33:47 +0530
Subject: [PATCH 232/439] packaging: include PEP 561 marker in wheels

---
 pyproject.toml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index 52e844b..ea243f4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -107,6 +107,9 @@ package-dir = {"" = "src"}
 where = ["src"]
 include = ["framevitals*"]
 
+[tool.setuptools.package-data]
+framevitals = ["py.typed"]
+
 [tool.maturin]
 python-source = "src"
 module-name = "framevitals._native"

From 19761df7f6e91faf79eee8b1332a6ca401fbfa3b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:34:02 +0530
Subject: [PATCH 233/439] ci: require PEP 561 marker in wheel

---
 .github/workflows/package.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index be0fda2..786dc2d 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -44,6 +44,9 @@ jobs:
           assert any(name.startswith("framevitals/") for name in names), (
               "framevitals package missing from wheel"
           )
+          assert "framevitals/py.typed" in names, (
+              "PEP 561 marker missing from wheel"
+          )
           assert not any(name.startswith("modules/") for name in names), (
               "legacy modules package leaked into wheel"
           )

From 1d6df7d8511883da3290a6010be19fff05d64ab7 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:34:31 +0530
Subject: [PATCH 234/439] docs: capture source-aware overhaul in changelog

---
 CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 41 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 181e82a..fdb8a76 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,13 +6,50 @@ FrameVitals follows semantic versioning while the public API matures. The 0.x se
 
 ## Unreleased
 
-Development continues on the `dev` branch. Changes intended for the next release should be documented here before they are promoted to `main`.
+Development continues on the development branches. Changes intended for the next release should be documented here before they are promoted to `main`.
 
 ### Added
 
-- Data-contract inference through `framevitals.infer_contract()`
-- Structured contract validation through `framevitals.validate()`
-- `framevitals infer-contract` and `framevitals validate` CLI commands for CI-friendly data gates
+- Data-contract inference through `framevitals.infer_contract()` with versioned contract schemas, tolerant null-rate and numeric-bound expectations, categorical domains, optional-column semantics, and uniqueness hints
+- Structured exact contract validation through `framevitals.validate()`
+- `framevitals.gate()` for one CI-friendly pass/warn/fail verdict combining validation and drift
+- `framevitals gate` CLI command with configurable drift warning/failure thresholds, optional contract enforcement, JSON output, and CI exit codes
+- `framevitals plan` for previewing applicable modules, execution budgets, and source planning without running the heavy pipeline
+- Compact versioned analysis snapshots with deterministic fingerprints, JSON persistence, and snapshot-to-snapshot schema/health/ML-readiness/finding diffs
+- Dict-compatible result objects including `AnalysisResult`, `DriftResult`, `ValidationResult`, `GateResult`, and per-column result views
+- Focused public APIs for profiling, column roles, health, ML readiness, quality, statistics, anomalies, relationships, and target analysis without running unrelated pipeline stages
+- Source abstraction with metadata inspection, projection, and streaming capability discovery
+- Bounded Arrow-backed Parquet execution for public profiling, health, ML-readiness, roles, quality, statistics, anomaly, relationship, target-aware, drift, gate, planning, and full-analysis paths where those diagnostics support bounded execution
+- Optional Arrow-backed CSV and TSV streaming with exact row-count metadata, projected record batches, and pandas fallback for incompatible inputs
+- Execution provenance that distinguishes exact full-stream metrics, bounded row samples, estimates, and full materialization
+- Optional Rust native core and Python bridge for accelerated streaming numeric state, cardinality/quantile sketches, and backend routing
+- Dedicated Arrow/NumPy-fallback and Rust/native CI coverage
+- Benchmark harnesses for scale-sensitive profiling behavior
+- PEP 561 `py.typed` marker for downstream type-checking support
+- Optional `excel` dependency group for XLS/XLSX readers
+
+### Changed
+
+- Full `framevitals.analyze()` now dispatches streaming-capable sources through a bounded source-aware pipeline when artifacts do not require materialization
+- Drift comparison now preserves exact source shape metadata while bounding value-distribution work for streaming sources
+- `framevitals.gate()` reuses the same canonical drift and validation engines instead of maintaining separate data-loading logic
+- Contract validation explicitly remains exact for uniqueness, allowed-value, nullability, and bound constraints rather than silently weakening them to sampled checks
+- `framevitals.api` is now a lazy compatibility facade over the canonical focused, analysis, planning, and operations engines instead of a second eager implementation
+- The installed CLI now routes directly through canonical APIs and no longer relies on a runtime monkeypatch shim
+- CLI `--version` works from a no-dependencies wheel smoke install without importing the analytics stack
+- The `all` extra now actually includes the Arrow capability
+- Excel engines moved out of the default dependency set into `framevitals[excel]`; AI-only Pydantic moved into `framevitals[ai]`
+- Excel loading now reports a FrameVitals-specific install hint when the optional reader capability is missing
+- README and roadmap now describe the source-aware analyze → compare → validate → gate → monitor direction and document actual CLI exit semantics
+- Package CI now requires the `py.typed` marker to be present in built wheels
+
+### Fixed
+
+- Streaming ML-readiness results now disclose when duplicate rate is estimated from a bounded sample instead of reporting sampled data as exact
+- Streaming relationship metadata reports true source size instead of sample size
+- Streaming column-role cardinality metadata distinguishes bounded-sample cardinality from full-stream native estimates
+- CSV/TSV streaming sources preserve `FileSource` compatibility for existing callers and tests
+- Dedicated Arrow CI now exercises streaming statistics, drift, CSV/TSV sources, and the quality-gate CLI
 
 ## 0.1.0 - 2026-08-15
 

From fd35aec03823d6c5b361dede570a759eecf41149 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:34:55 +0530
Subject: [PATCH 235/439] ci: cancel stale package runs and restrict
 permissions

---
 .github/workflows/package.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index 786dc2d..e28d687 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -5,6 +5,13 @@ on:
     branches: [main, dev, "develop/august"]
   pull_request:
 
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     runs-on: ubuntu-latest

From f89939e4eca4d55df03df8172962a2c60c135027 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:35:16 +0530
Subject: [PATCH 236/439] ci: cancel stale test runs and restrict permissions

---
 .github/workflows/test.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2d9888f..9d57087 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -5,6 +5,13 @@ on:
     branches: [main, dev, "develop/august"]
   pull_request:
 
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   quality:
     name: package quality

From 7b4e8582cc3657e93226dd603d706052a0ed3919 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:35:37 +0530
Subject: [PATCH 237/439] ci: verify release version before PyPI publish

---
 .github/workflows/release.yml | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 588cdae..92c5999 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -4,6 +4,9 @@ on:
   release:
     types: [published]
 
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Build distribution
@@ -25,6 +28,32 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install build twine
 
+      - name: Verify release version
+        env:
+          RELEASE_TAG: ${{ github.event.release.tag_name }}
+        run: |
+          PYTHONPATH=src python - <<'PY'
+          import os
+          import tomllib
+          from pathlib import Path
+
+          import framevitals
+
+          tag = os.environ["RELEASE_TAG"]
+          release_version = tag[1:] if tag.startswith("v") else tag
+          with Path("pyproject.toml").open("rb") as handle:
+              project_version = tomllib.load(handle)["project"]["version"]
+          package_version = framevitals.__version__
+
+          assert release_version == project_version, (
+              f"release tag {tag!r} != pyproject version {project_version!r}"
+          )
+          assert project_version == package_version, (
+              f"pyproject version {project_version!r} != package version {package_version!r}"
+          )
+          print(f"Release version verified: {project_version}")
+          PY
+
       - name: Build distributions
         run: python -m build
 
@@ -46,6 +75,7 @@ jobs:
       url: https://pypi.org/p/framevitals
     permissions:
       id-token: write
+      contents: read
 
     steps:
       - name: Download distributions

From edaf3a44de572e028ad3678be3982fc96fb01884 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:35:58 +0530
Subject: [PATCH 238/439] ci: add reproducible performance benchmark workflow

---
 .github/workflows/benchmark.yml | 89 +++++++++++++++++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 .github/workflows/benchmark.yml

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 0000000..e6fba7a
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,89 @@
+name: Performance Benchmark
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      rows:
+        description: Rows in the synthetic benchmark workload
+        required: false
+        default: "50000"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  profile-scale:
+    name: profile scale benchmark
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install benchmark runtime
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[arrow]"
+
+      - name: Run reproducible profile benchmark
+        env:
+          BENCHMARK_ROWS: ${{ inputs.rows || '50000' }}
+        run: |
+          python benchmarks/benchmark_profile_scale.py \
+            --rows "$BENCHMARK_ROWS" \
+            --numeric-columns 40 \
+            --categorical-columns 5 \
+            --scenarios numpy auto parquet \
+            --output benchmark-results.json
+
+      - name: Publish benchmark summary
+        run: |
+          python - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          payload = json.loads(Path("benchmark-results.json").read_text(encoding="utf-8"))
+          lines = [
+              "## FrameVitals profile benchmark",
+              "",
+              "| Scenario | Seconds | Peak RSS (MB) | Backend |",
+              "| --- | ---: | ---: | --- |",
+          ]
+          for measurement in payload["measurements"]:
+              backend = measurement.get("backend_status", {}).get("selected", "unknown")
+              lines.append(
+                  f"| {measurement['scenario']} | {measurement['elapsed_seconds']:.3f} "
+                  f"| {measurement['peak_rss_mb']:.1f} | {backend} |"
+              )
+          lines.extend([
+              "",
+              "Results are measurements, not absolute pass/fail thresholds. Compare like-for-like runs.",
+          ])
+          Path(os.environ["GITHUB_STEP_SUMMARY"]).write_text(
+              "\n".join(lines) + "\n",
+              encoding="utf-8",
+          )
+          PY
+
+      - name: Upload benchmark result
+        uses: actions/upload-artifact@v5
+        with:
+          name: framevitals-profile-benchmark
+          path: benchmark-results.json
+          retention-days: 30

From 02e32cade0c4810400a8b43a8195d5c061e88c8d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:37:20 +0530
Subject: [PATCH 239/439] refactor: load plotting modules only when needed

---
 src/framevitals/pipeline.py | 55 +++++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 20 deletions(-)

diff --git a/src/framevitals/pipeline.py b/src/framevitals/pipeline.py
index c655335..0486af0 100644
--- a/src/framevitals/pipeline.py
+++ b/src/framevitals/pipeline.py
@@ -30,7 +30,6 @@
 from framevitals.config import VALID_MODULES
 from framevitals.dataset_signals import detect_dataset_signals
 from framevitals.execution import derive_execution_budget
-from framevitals.explainability import explain_winner
 from framevitals.health_score import calculate_health_score
 from framevitals.loader import load_dataset
 from framevitals.ml_readiness import calculate_ml_readiness
@@ -40,7 +39,6 @@
 from framevitals.signal_engine import build_signals
 from framevitals.target_intelligence import run_target_intelligence
 from framevitals.text_profile import profile_text_columns
-from framevitals.visualizer import generate_charts
 
 
 logger = logging.getLogger("framevitals.pipeline")
@@ -313,14 +311,20 @@ def module_enabled(name: str) -> bool:
     if winner_available:
         if module_enabled("explainability"):
             t0 = time.perf_counter()
-            _, explainability, _ = _safe_call(
-                "explainability",
-                lambda: explain_winner(
+
+            def run_explainability():
+                from framevitals.explainability import explain_winner
+
+                return explain_winner(
                     df,
                     target_column=target_column,
                     leaderboard_result=model_leaderboard,
                     dataset_id=dataset_id,
-                ),
+                )
+
+            _, explainability, _ = _safe_call(
+                "explainability",
+                run_explainability,
             )
             timings_ms["explainability"] = (time.perf_counter() - t0) * 1000
             module_status["explainability"] = _result_status(explainability)
@@ -375,20 +379,31 @@ def module_enabled(name: str) -> bool:
     charts_applicable = write_artifacts and analysis_mode in {"standard", "deep", "research"}
     if charts_applicable and module_enabled("charts"):
         t0 = time.perf_counter()
-        charts = generate_charts(
-            dataset_id,
-            df,
-            health,
-            advanced,
-            cleaning,
-            target_column=target_column,
-            model_leaderboard=model_leaderboard,
-            explainability=explainability,
-            time_series=time_series_analysis,
-            deep_statistics_v2=deep_statistics_v2,
-        )
-        timings_ms["charts"] = (time.perf_counter() - t0) * 1000
-        module_status["charts"] = "ran"
+
+        def render_charts():
+            from framevitals.visualizer import generate_charts
+
+            return generate_charts(
+                dataset_id,
+                df,
+                health,
+                advanced,
+                cleaning,
+                target_column=target_column,
+                model_leaderboard=model_leaderboard,
+                explainability=explainability,
+                time_series=time_series_analysis,
+                deep_statistics_v2=deep_statistics_v2,
+            )
+
+        _, rendered_charts, chart_elapsed = _safe_call("charts", render_charts)
+        timings_ms["charts"] = chart_elapsed
+        if isinstance(rendered_charts, list):
+            charts = rendered_charts
+            module_status["charts"] = "ran"
+        else:
+            charts = []
+            module_status["charts"] = _result_status(rendered_charts)
     elif not module_enabled("charts"):
         timings_ms["charts"] = 0.0
         module_status["charts"] = "disabled"

From c6819a9fe82e00a8c90a13a9863a65ce3afe16bf Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:39:46 +0530
Subject: [PATCH 240/439] refactor: isolate optional web dependencies and
 canonical drift

---
 app.py | 127 ++++++++++++++++++++++++++++++++-------------------------
 1 file changed, 71 insertions(+), 56 deletions(-)

diff --git a/app.py b/app.py
index d022c28..f202231 100644
--- a/app.py
+++ b/app.py
@@ -5,39 +5,21 @@
 server-rendered report routes available for local use.
 """
 
-import os
 import math
-from pathlib import Path
+import os
 from copy import deepcopy
-from time import perf_counter
+from pathlib import Path
 from threading import Lock, Thread
+from time import perf_counter
 
-from flask import Flask, render_template, request, redirect, url_for, send_file, session, jsonify
+from flask import Flask, jsonify, redirect, render_template, request, send_file, session, url_for
 from werkzeug.exceptions import ClientDisconnected
 
-from framevitals.loader import (
-    load_dataset,
-    save_uploaded_file,
-)
-from framevitals.pipeline import (
-    run_full_analysis,
-)
-from framevitals.ai_insights import (
-    answer_dataset_question,
-)
-from framevitals.ai_agent import (
-    answer_with_agent,
-)
-from framevitals.report_generator import (
-    generate_pdf_report,
-)
-from framevitals.frontend_api import (
-    build_dashboard_payload,
-)
-from framevitals.drift_analysis import (
-    compare_datasets,
-    split_by_date,
-)
+from framevitals.ai_insights import answer_dataset_question
+from framevitals.drift_analysis import split_by_date
+from framevitals.frontend_api import build_dashboard_payload
+from framevitals.loader import load_dataset, save_uploaded_file
+from framevitals.pipeline import run_full_analysis
 
 
 # ---------------------------------------------------------------------------
@@ -48,7 +30,6 @@
 # JSON.parse both do), which makes the frontend silently fall back to an
 # empty payload. We walk every payload recursively and replace those values
 # with None so the wire format is RFC-8259 compliant.
-
 def _is_nonfinite(v) -> bool:
     return isinstance(v, float) and not math.isfinite(v)
 
@@ -73,7 +54,7 @@ def safe_jsonify(payload):
 app = Flask(__name__)
 app.secret_key = os.environ.get(
     "FRAMEVITALS_SECRET_KEY",
-    "development-only-secret"
+    "development-only-secret",
 )
 app.config["MAX_CONTENT_LENGTH"] = 50 * 1024 * 1024
 
@@ -90,6 +71,7 @@ def safe_jsonify(payload):
 
 class DotDict(dict):
     """Allow dict.key access for Jinja templates."""
+
     __getattr__ = dict.get
     __setattr__ = dict.__setitem__
     __delattr__ = dict.__delitem__
@@ -112,7 +94,12 @@ def _get_report_job(dataset_id: str) -> dict:
         return dict(REPORT_JOBS.get(dataset_id, {}))
 
 
-def _set_report_job(dataset_id: str, status: str, pdf_path: Path | None = None, error: str | None = None) -> dict:
+def _set_report_job(
+    dataset_id: str,
+    status: str,
+    pdf_path: Path | None = None,
+    error: str | None = None,
+) -> dict:
     job = {
         "status": status,
         "pdf_path": str(pdf_path) if pdf_path else None,
@@ -142,6 +129,10 @@ def _queue_pdf_generation(dataset_id: str, result: dict | None = None) -> dict:
     def worker():
         _set_report_job(dataset_id, "running")
         try:
+            # PDF/report dependencies are intentionally optional for the Flask
+            # runtime. Import them only when a report is actually requested.
+            from framevitals.report_generator import generate_pdf_report
+
             pdf_path = generate_pdf_report(deepcopy(cached_result))
             _set_report_job(dataset_id, "ready", pdf_path=pdf_path)
         except Exception as exc:
@@ -198,7 +189,10 @@ def analyze():
         except ClientDisconnected:
             return render_template(
                 "error.html",
-                message="The upload was interrupted before Flask finished reading the file. Please try again.",
+                message=(
+                    "The upload was interrupted before Flask finished reading the file. "
+                    "Please try again."
+                ),
             ), 400
 
         if not uploaded_file or uploaded_file.filename == "":
@@ -220,7 +214,7 @@ def analyze():
         with REPORT_LOCK:
             ANALYSIS_CACHE[dataset_id] = deepcopy(result)
 
-        report_job = _queue_pdf_generation(dataset_id, result)
+        _queue_pdf_generation(dataset_id, result)
         result["report_status"] = _report_status_payload(dataset_id)
 
         session["dataset_id"] = dataset_id
@@ -229,13 +223,12 @@ def analyze():
         session["analysis_mode"] = analysis_mode
         session["target_column"] = target_column
 
-        # Convert to DotDict for Jinja template dot-access
         result_dot = DotDict.from_dict(result)
-
         return render_template("report.html", result=result_dot)
 
     except Exception as exc:
         import traceback
+
         traceback.print_exc()
         return render_template("error.html", message=str(exc))
 
@@ -249,7 +242,12 @@ def api_analyze():
             analysis_mode = request.form.get("analysis_mode", "standard")
             target_column = request.form.get("target_column") or None
         except ClientDisconnected:
-            return jsonify({"error": "The upload was interrupted before the server finished reading it. Please try again."}), 400
+            return jsonify({
+                "error": (
+                    "The upload was interrupted before the server finished reading it. "
+                    "Please try again."
+                )
+            }), 400
 
         if analysis_mode not in {"quick", "standard", "deep", "research"}:
             analysis_mode = "standard"
@@ -331,9 +329,11 @@ def ask():
         _queue_pdf_generation(dataset_id, result)
         result["report_status"] = _report_status_payload(dataset_id)
 
-        # Try agentic answer first; fall back to the legacy single-shot answerer
-        # if anything goes wrong (Ollama offline, model errors, etc.).
+        # Agentic AI is an optional capability. If it is not installed or the
+        # model is unavailable, fall back to the lightweight answerer.
         try:
+            from framevitals.ai_agent import answer_with_agent
+
             agent_response = answer_with_agent(
                 question=question,
                 df=load_dataset(Path(file_path)),
@@ -375,7 +375,6 @@ def api_ask():
         if not question:
             return jsonify({"error": "Missing 'question' in request body."}), 400
 
-        # Look up the cached analysis result first
         with REPORT_LOCK:
             cached_result = ANALYSIS_CACHE.get(dataset_id)
 
@@ -384,7 +383,6 @@ def api_ask():
                 "error": "No cached analysis was found for this dataset. Run /api/analyze first.",
             }), 404
 
-        # Reload dataframe (cheap; uploaded files live under uploads/)
         file_path = session.get("file_path")
         df = None
         if file_path:
@@ -394,9 +392,8 @@ def api_ask():
                 df = None
 
         try:
-            # Fast mode by default (single writer call, no critic/repair).
-            # Pass {"mode": "full"} in the body to opt back in to the full
-            # planner→executor→critic→writer loop.
+            from framevitals.ai_agent import answer_with_agent
+
             mode = (body.get("mode") or "fast").lower().strip()
             response = answer_with_agent(
                 question=question,
@@ -413,7 +410,11 @@ def api_ask():
                 ml_readiness=cached_result["ml_readiness"],
                 advanced=cached_result.get("advanced"),
             )
-            response = {"source": response.get("source", "fallback"), "answer": response.get("answer", str(exc)), "trace": {}}
+            response = {
+                "source": response.get("source", "fallback"),
+                "answer": response.get("answer", str(exc)),
+                "trace": {},
+            }
 
         return safe_jsonify({
             "question": question,
@@ -425,6 +426,7 @@ def api_ask():
 
     except Exception as exc:
         import traceback
+
         traceback.print_exc()
         return jsonify({"error": str(exc)}), 500
 
@@ -450,11 +452,11 @@ def api_ai_report():
             cached = ANALYSIS_CACHE.get(dataset_id)
 
         if cached is None:
-            return jsonify({"error": "No cached analysis for this dataset. Re-run /api/analyze."}), 404
+            return jsonify({
+                "error": "No cached analysis for this dataset. Re-run /api/analyze."
+            }), 404
 
-        from framevitals.ai_insights import (
-            generate_ai_report,
-        )
+        from framevitals.ai_insights import generate_ai_report
 
         try:
             ai_report = generate_ai_report(
@@ -469,7 +471,6 @@ def api_ai_report():
         except Exception as exc:
             ai_report = {"source": f"error: {exc}", "text": str(exc)}
 
-        # Re-cache so subsequent /api/analyze calls (or PDF regen) include it.
         with REPORT_LOCK:
             cached["ai_report"] = ai_report
             ANALYSIS_CACHE[dataset_id] = cached
@@ -478,6 +479,7 @@ def api_ai_report():
 
     except Exception as exc:
         import traceback
+
         traceback.print_exc()
         return jsonify({"error": str(exc)}), 500
 
@@ -539,19 +541,23 @@ def api_compare():
         _, ref_path, _ = save_uploaded_file(ref_file)
         _, cur_path, _ = save_uploaded_file(cur_file)
 
-        df_ref = load_dataset(ref_path)
-        df_cur = load_dataset(cur_path)
-
         columns_param = request.form.get("columns", "").strip()
-        columns = [c.strip() for c in columns_param.split(",") if c.strip()] if columns_param else None
+        columns = (
+            [c.strip() for c in columns_param.split(",") if c.strip()]
+            if columns_param
+            else None
+        )
 
-        report = compare_datasets(df_ref, df_cur, columns=columns)
+        from framevitals.operations import compare
+
+        report = compare(ref_path, cur_path, columns=columns)
         report["reference_filename"] = ref_file.filename
         report["current_filename"] = cur_file.filename
         return safe_jsonify(report)
 
     except Exception as exc:
         import traceback
+
         traceback.print_exc()
         return jsonify({"error": str(exc)}), 500
 
@@ -592,7 +598,9 @@ def api_compare_self():
         except ValueError as exc:
             return jsonify({"error": str(exc)}), 400
 
-        report = compare_datasets(df_ref, df_cur)
+        from framevitals.operations import compare
+
+        report = compare(df_ref, df_cur)
         report["reference_filename"] = f"{ds_name} (older {ratio:.0%})"
         report["current_filename"] = f"{ds_name} (newer {1 - ratio:.0%})"
         report["split_by"] = date_column
@@ -601,6 +609,7 @@ def api_compare_self():
 
     except Exception as exc:
         import traceback
+
         traceback.print_exc()
         return jsonify({"error": str(exc)}), 500
 
@@ -639,11 +648,17 @@ def download_report(dataset_id):
         if report_status["ready"] and pdf_path.exists() and pdf_path.stat().st_size > 0:
             return send_file(pdf_path, as_attachment=True)
 
-        message = "The PDF report is generating in the background. Please try again in a few seconds."
+        message = (
+            "The PDF report is generating in the background. "
+            "Please try again in a few seconds."
+        )
         if report_status["status"] == "failed":
             message = f"PDF generation failed: {report_status.get('error', 'Unknown error')}"
         elif result is None:
-            message = "No cached analysis was found for this dataset. Please run analysis again first."
+            message = (
+                "No cached analysis was found for this dataset. "
+                "Please run analysis again first."
+            )
 
         return render_template("error.html", message=message), 202
 

From 4e667a6c7259705c41eef0957fac002e66ee177e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:40:06 +0530
Subject: [PATCH 241/439] test: lock lazy web optional dependencies

---
 tests/test_web_dependency_boundaries.py | 33 +++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 tests/test_web_dependency_boundaries.py

diff --git a/tests/test_web_dependency_boundaries.py b/tests/test_web_dependency_boundaries.py
new file mode 100644
index 0000000..3c28f1d
--- /dev/null
+++ b/tests/test_web_dependency_boundaries.py
@@ -0,0 +1,33 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def test_web_app_does_not_eagerly_import_optional_ai_or_report_stack():
+    repo_root = Path(__file__).resolve().parents[1]
+    env = os.environ.copy()
+    env["PYTHONPATH"] = str(repo_root / "src")
+    completed = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import sys; import app; "
+                "assert 'framevitals.ai_agent' not in sys.modules; "
+                "assert 'pydantic' not in sys.modules; "
+                "assert 'framevitals.report_generator' not in sys.modules; "
+                "assert 'framevitals.pdf_report_builder' not in sys.modules; "
+                "assert 'framevitals.visualizer' not in sys.modules; "
+                "assert 'framevitals.explainability' not in sys.modules; "
+                "assert 'matplotlib' not in sys.modules; "
+                "assert 'seaborn' not in sys.modules"
+            ),
+        ],
+        cwd=repo_root,
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert completed.returncode == 0, completed.stderr

From e9624fabfe8a48a19eb24430bc57c4ed4c940b2a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:40:59 +0530
Subject: [PATCH 242/439] refactor: make explainability plotting optional

---
 src/framevitals/explainability.py | 220 +++++++++++++++++++-----------
 1 file changed, 142 insertions(+), 78 deletions(-)

diff --git a/src/framevitals/explainability.py b/src/framevitals/explainability.py
index a66a081..07d396d 100644
--- a/src/framevitals/explainability.py
+++ b/src/framevitals/explainability.py
@@ -9,7 +9,7 @@
        For linear models, use shap.LinearExplainer.
        Fallback: sklearn permutation importance (model-agnostic).
     3. Collapse one-hot expansions back to original feature names.
-    4. Save a beeswarm summary plot to outputs/charts.
+    4. Save a beeswarm summary plot when plotting support is installed.
 
 Public entry point:
     explain_winner(df, target_column, leaderboard_result) -> dict
@@ -21,13 +21,8 @@
 from pathlib import Path
 from typing import Any
 
-import matplotlib
-
-matplotlib.use("Agg")
-import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-
 from sklearn.dummy import DummyClassifier, DummyRegressor
 from sklearn.ensemble import (
     GradientBoostingClassifier,
@@ -50,29 +45,38 @@
 
 CHART_DIR = Path("static/charts")
 
+
 # ---------------------------------------------------------------------------
 # Estimator rebuilds (mirrors model_leaderboard registry)
 # ---------------------------------------------------------------------------
-
 def _rebuild_estimator(name: str, task_type: str):
     """Recreate the estimator by name. Returns None if unknown/missing."""
-    name_lower = name.lower()
-
     if task_type == "classification":
         if name == "DummyClassifier":
             return DummyClassifier(strategy="most_frequent")
         if name == "LogisticRegression":
             return LogisticRegression(
-                max_iter=2000, n_jobs=-1, class_weight="balanced", random_state=42
+                max_iter=2000,
+                n_jobs=-1,
+                class_weight="balanced",
+                random_state=42,
             )
         if name == "KNeighborsClassifier":
             return KNeighborsClassifier(n_neighbors=7)
         if name == "RandomForestClassifier":
             return RandomForestClassifier(
-                n_estimators=200, max_depth=10, random_state=42, n_jobs=-1, class_weight="balanced"
+                n_estimators=200,
+                max_depth=10,
+                random_state=42,
+                n_jobs=-1,
+                class_weight="balanced",
             )
         if name == "GradientBoostingClassifier":
-            return GradientBoostingClassifier(n_estimators=150, max_depth=4, random_state=42)
+            return GradientBoostingClassifier(
+                n_estimators=150,
+                max_depth=4,
+                random_state=42,
+            )
         if name == "XGBClassifier":
             try:
                 from xgboost import XGBClassifier
@@ -113,9 +117,18 @@ def _rebuild_estimator(name: str, task_type: str):
         if name == "KNeighborsRegressor":
             return KNeighborsRegressor(n_neighbors=7)
         if name == "RandomForestRegressor":
-            return RandomForestRegressor(n_estimators=200, max_depth=10, random_state=42, n_jobs=-1)
+            return RandomForestRegressor(
+                n_estimators=200,
+                max_depth=10,
+                random_state=42,
+                n_jobs=-1,
+            )
         if name == "GradientBoostingRegressor":
-            return GradientBoostingRegressor(n_estimators=150, max_depth=4, random_state=42)
+            return GradientBoostingRegressor(
+                n_estimators=150,
+                max_depth=4,
+                random_state=42,
+            )
         if name == "XGBRegressor":
             try:
                 from xgboost import XGBRegressor
@@ -168,7 +181,6 @@ def _is_linear_model(name: str) -> bool:
 # ---------------------------------------------------------------------------
 # One-hot collapse
 # ---------------------------------------------------------------------------
-
 def _collapse_to_original(
     feature_names: list[str],
     importances: np.ndarray,
@@ -176,13 +188,14 @@ def _collapse_to_original(
     categorical_features: list[str],
 ) -> list[dict]:
     """Sum one-hot expansions back to their original column names."""
-    collapsed: dict[str, float] = {col: 0.0 for col in numeric_features + categorical_features}
+    collapsed: dict[str, float] = {
+        col: 0.0 for col in numeric_features + categorical_features
+    }
 
     for fname, value in zip(feature_names, importances):
         if fname in collapsed:
             collapsed[fname] += float(value)
             continue
-        # OneHot pattern: "<col>_<level>"
         matched = False
         for col in categorical_features:
             if fname.startswith(col + "_"):
@@ -193,27 +206,34 @@ def _collapse_to_original(
             collapsed[fname] = float(value)
 
     rows = [
-        {"feature": k, "importance": round(v, 6)} for k, v in collapsed.items()
+        {"feature": key, "importance": round(value, 6)}
+        for key, value in collapsed.items()
     ]
-    rows.sort(key=lambda r: abs(r["importance"]), reverse=True)
+    rows.sort(key=lambda row: abs(row["importance"]), reverse=True)
     return rows
 
 
 # ---------------------------------------------------------------------------
-# SHAP plots
+# Optional SHAP plot
 # ---------------------------------------------------------------------------
-
 def _save_summary_plot(
-    shap_values, X_transformed, feature_names, dataset_id: str
+    shap_values,
+    X_transformed,
+    feature_names,
+    dataset_id: str,
 ) -> str | None:
+    """Write a SHAP summary plot when optional plotting support is available."""
     try:
-        import shap
+        import matplotlib
 
-        CHART_DIR.mkdir(
-            parents=True,
-            exist_ok=True,
-        )
+        matplotlib.use("Agg")
+        import matplotlib.pyplot as plt
+        import shap
+    except Exception:
+        return None
 
+    try:
+        CHART_DIR.mkdir(parents=True, exist_ok=True)
         plt.figure(figsize=(9, 6))
         shap.summary_plot(
             shap_values,
@@ -225,17 +245,16 @@ def _save_summary_plot(
         path = CHART_DIR / f"{dataset_id}_shap_summary.png"
         plt.tight_layout()
         plt.savefig(path, dpi=150, bbox_inches="tight")
-        plt.close()
         return str(path)
     except Exception:
-        plt.close("all")
         return None
+    finally:
+        plt.close("all")
 
 
 # ---------------------------------------------------------------------------
 # Permutation importance fallback
 # ---------------------------------------------------------------------------
-
 def _permutation_importance_block(
     pipeline: Pipeline,
     X: pd.DataFrame,
@@ -247,17 +266,23 @@ def _permutation_importance_block(
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
             result = permutation_importance(
-                pipeline, X, y, n_repeats=5, random_state=42, n_jobs=-1, scoring=None
+                pipeline,
+                X,
+                y,
+                n_repeats=5,
+                random_state=42,
+                n_jobs=-1,
+                scoring=None,
             )
         rows = [
             {
                 "feature": col,
-                "importance": round(float(result.importances_mean[i]), 6),
-                "std": round(float(result.importances_std[i]), 6),
+                "importance": round(float(result.importances_mean[index]), 6),
+                "std": round(float(result.importances_std[index]), 6),
             }
-            for i, col in enumerate(X.columns)
+            for index, col in enumerate(X.columns)
         ]
-        rows.sort(key=lambda r: abs(r["importance"]), reverse=True)
+        rows.sort(key=lambda row: abs(row["importance"]), reverse=True)
         return {"available": True, "method": "permutation", "top_features": rows[:15]}
     except Exception as exc:
         return {"available": False, "reason": str(exc)}
@@ -266,7 +291,6 @@ def _permutation_importance_block(
 # ---------------------------------------------------------------------------
 # Public entry point
 # ---------------------------------------------------------------------------
-
 def explain_winner(
     df: pd.DataFrame,
     target_column: str,
@@ -277,16 +301,8 @@ def explain_winner(
     """
     Generate SHAP-based global + per-row explanations for the winning model.
 
-    Args:
-        df: Original dataframe.
-        target_column: Target column name.
-        leaderboard_result: Output of run_model_leaderboard.
-        dataset_id: Used for the SHAP summary chart filename.
-        sample_size: Number of test rows to compute SHAP on (cap for speed).
-
-    Returns:
-        JSON-safe dict with method, global ranking, sample per-row stories,
-        and the path to the SHAP summary chart (if produced).
+    Plot generation is optional. The structured explanation and permutation
+    fallback remain usable without Matplotlib/Seaborn.
     """
     if not leaderboard_result.get("available") or not leaderboard_result.get("winner"):
         return {"available": False, "message": "No leaderboard winner to explain."}
@@ -296,11 +312,17 @@ def explain_winner(
 
     estimator = _rebuild_estimator(winner_name, task_type)
     if estimator is None:
-        return {"available": False, "message": f"Could not rebuild estimator: {winner_name}"}
+        return {
+            "available": False,
+            "message": f"Could not rebuild estimator: {winner_name}",
+        }
 
     prep = prepare_ml_matrix(df, target=target_column)
     if not prep["usable"]:
-        return {"available": False, "message": "Preprocessing produced no usable features."}
+        return {
+            "available": False,
+            "message": "Preprocessing produced no usable features.",
+        }
 
     X = prep["X"]
     y = prep["y"]
@@ -310,21 +332,39 @@ def explain_winner(
     numeric_features = prep["numeric_features"]
     categorical_features = prep["categorical_features"]
 
-    preprocessor = build_sklearn_preprocessor(numeric_features, categorical_features)
+    preprocessor = build_sklearn_preprocessor(
+        numeric_features,
+        categorical_features,
+    )
     pipeline = Pipeline([("pre", preprocessor), ("model", estimator)])
 
-    # Fit once on a 75/25 split so we have a held-out slice for SHAP
     try:
+        stratify = (
+            y
+            if (
+                task_type == "classification"
+                and y.nunique() > 1
+                and y.value_counts().min() >= 2
+            )
+            else None
+        )
         X_train, X_test, y_train, y_test = train_test_split(
-            X, y, test_size=0.25, random_state=42,
-            stratify=y if task_type == "classification" and y.nunique() > 1 and y.value_counts().min() >= 2 else None,
+            X,
+            y,
+            test_size=0.25,
+            random_state=42,
+            stratify=stratify,
         )
     except Exception:
-        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)
+        X_train, X_test, y_train, y_test = train_test_split(
+            X,
+            y,
+            test_size=0.25,
+            random_state=42,
+        )
 
     pipeline.fit(X_train, y_train)
 
-    # Sample the test set for SHAP speed
     if len(X_test) > sample_size:
         X_test_sample = X_test.sample(sample_size, random_state=42)
     else:
@@ -333,7 +373,11 @@ def explain_winner(
     fitted_pre = pipeline.named_steps["pre"]
     fitted_model = pipeline.named_steps["model"]
     X_test_transformed = fitted_pre.transform(X_test_sample)
-    feature_names = get_transformed_feature_names(fitted_pre, numeric_features, categorical_features)
+    feature_names = get_transformed_feature_names(
+        fitted_pre,
+        numeric_features,
+        categorical_features,
+    )
 
     method = None
     summary_chart_path: str | None = None
@@ -341,7 +385,6 @@ def explain_winner(
     per_row_stories: list[dict] = []
     error_messages: list[str] = []
 
-    # Try SHAP
     try:
         import shap
 
@@ -353,25 +396,27 @@ def explain_winner(
                 shap_values_raw = explainer.shap_values(X_test_transformed)
                 method = "shap.TreeExplainer"
             elif _is_linear_model(winner_name):
-                # LinearExplainer needs a background dataset
-                bg = fitted_pre.transform(X_train.sample(min(100, len(X_train)), random_state=42))
-                explainer = shap.LinearExplainer(fitted_model, bg)
+                background = fitted_pre.transform(
+                    X_train.sample(min(100, len(X_train)), random_state=42)
+                )
+                explainer = shap.LinearExplainer(fitted_model, background)
                 shap_values_raw = explainer.shap_values(X_test_transformed)
                 method = "shap.LinearExplainer"
             else:
-                explainer = None
                 shap_values_raw = None
 
         if shap_values_raw is not None:
-            # For multiclass tree explainers, shap returns a list per class -> use mean abs across classes
             if isinstance(shap_values_raw, list):
-                shap_values = np.mean([np.abs(arr) for arr in shap_values_raw], axis=0)
-                # Signed values for plotting: take positive class or mean across classes
+                shap_values = np.mean(
+                    [np.abs(array) for array in shap_values_raw],
+                    axis=0,
+                )
                 shap_values_for_plot = (
-                    shap_values_raw[1] if len(shap_values_raw) == 2 else shap_values_raw[0]
+                    shap_values_raw[1]
+                    if len(shap_values_raw) == 2
+                    else shap_values_raw[0]
                 )
             elif shap_values_raw.ndim == 3:
-                # newer SHAP returns (n_samples, n_features, n_classes)
                 shap_values = np.mean(np.abs(shap_values_raw), axis=2)
                 shap_values_for_plot = shap_values_raw[:, :, 0]
             else:
@@ -380,38 +425,57 @@ def explain_winner(
 
             mean_abs = shap_values.mean(axis=0)
             global_rows = _collapse_to_original(
-                feature_names, mean_abs, numeric_features, categorical_features
+                feature_names,
+                mean_abs,
+                numeric_features,
+                categorical_features,
             )
 
-            # Per-row stories: top-3 rows by total |shap|
             row_totals = shap_values.sum(axis=1)
             top_idx = np.argsort(-row_totals)[:3]
-            for i, ridx in enumerate(top_idx):
-                contributions = list(zip(feature_names, shap_values_for_plot[ridx]))
-                contributions.sort(key=lambda t: abs(t[1]), reverse=True)
+            for rank, row_index in enumerate(top_idx, start=1):
+                contributions = list(
+                    zip(feature_names, shap_values_for_plot[row_index])
+                )
+                contributions.sort(key=lambda item: abs(item[1]), reverse=True)
                 story = [
-                    {"feature": fname, "shap_value": round(float(val), 6)}
-                    for fname, val in contributions[:5]
+                    {
+                        "feature": feature,
+                        "shap_value": round(float(value), 6),
+                    }
+                    for feature, value in contributions[:5]
                 ]
+                source_index = X_test_sample.index[row_index]
                 per_row_stories.append({
-                    "rank": i + 1,
-                    "row_index": int(X_test_sample.index[ridx]) if hasattr(X_test_sample.index[ridx], "__int__") else str(X_test_sample.index[ridx]),
+                    "rank": rank,
+                    "row_index": (
+                        int(source_index)
+                        if hasattr(source_index, "__int__")
+                        else str(source_index)
+                    ),
                     "top_contributions": story,
                 })
 
             summary_chart_path = _save_summary_plot(
-                shap_values_for_plot, X_test_transformed, feature_names, dataset_id
+                shap_values_for_plot,
+                X_test_transformed,
+                feature_names,
+                dataset_id,
             )
 
     except Exception as exc:
         error_messages.append(f"SHAP failed: {exc}")
         method = None
 
-    # Permutation importance — always run as cross-check / fallback
-    perm = _permutation_importance_block(pipeline, X_test, y_test, numeric_features, categorical_features)
+    perm = _permutation_importance_block(
+        pipeline,
+        X_test,
+        y_test,
+        numeric_features,
+        categorical_features,
+    )
 
     if not global_rows and perm.get("available"):
-        # Use permutation as the global ranking
         global_rows = perm["top_features"]
         method = method or "permutation_importance"
 

From e7cde5fd9f08a37fa6179039718f3b506b6289b6 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:41:14 +0530
Subject: [PATCH 243/439] packaging: move plotting stack out of core

---
 pyproject.toml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index ea243f4..5ceb18e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,8 +35,6 @@ classifiers = [
 dependencies = [
     "pandas>=2.2",
     "numpy>=1.26",
-    "matplotlib>=3.8",
-    "seaborn>=0.13",
     "scipy>=1.13",
     "statsmodels>=0.14",
     "scikit-learn>=1.5",
@@ -47,6 +45,10 @@ excel = [
     "openpyxl>=3.1",
     "xlrd>=2.0",
 ]
+plot = [
+    "matplotlib>=3.8",
+    "seaborn>=0.13",
+]
 ml = [
     "xgboost>=2.0",
     "lightgbm>=4.3",
@@ -68,6 +70,8 @@ arrow = [
 all = [
     "openpyxl>=3.1",
     "xlrd>=2.0",
+    "matplotlib>=3.8",
+    "seaborn>=0.13",
     "xgboost>=2.0",
     "lightgbm>=4.3",
     "pyod>=2.0",
@@ -88,6 +92,8 @@ dev = [
     "ruff>=0.6",
     "openpyxl>=3.1",
     "xlrd>=2.0",
+    "matplotlib>=3.8",
+    "seaborn>=0.13",
     "pydantic>=2.7",
 ]
 

From 5099944d16966ca3d27e3c1310ff6b61da445a36 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:41:34 +0530
Subject: [PATCH 244/439] test: enforce lazy plotting dependency boundary

---
 tests/test_optional_dependency_boundaries.py | 40 ++++++++++++++++----
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/tests/test_optional_dependency_boundaries.py b/tests/test_optional_dependency_boundaries.py
index 2e3093e..93c34df 100644
--- a/tests/test_optional_dependency_boundaries.py
+++ b/tests/test_optional_dependency_boundaries.py
@@ -8,22 +8,48 @@
 from framevitals.loader import load_dataset
 
 
-def test_core_import_does_not_eagerly_load_ai_dependency():
+def _run_import_assertion(code: str):
     src_root = Path(__file__).resolve().parents[1] / "src"
     env = os.environ.copy()
     env["PYTHONPATH"] = str(src_root)
-    completed = subprocess.run(
-        [
-            sys.executable,
-            "-c",
-            "import sys; import framevitals; assert 'pydantic' not in sys.modules",
-        ],
+    return subprocess.run(
+        [sys.executable, "-c", code],
         cwd=src_root.parent,
         env=env,
         capture_output=True,
         text=True,
         check=False,
     )
+
+
+def test_core_import_does_not_eagerly_load_ai_dependency():
+    completed = _run_import_assertion(
+        "import sys; import framevitals; assert 'pydantic' not in sys.modules"
+    )
+    assert completed.returncode == 0, completed.stderr
+
+
+def test_pipeline_import_does_not_eagerly_load_plotting_stack():
+    completed = _run_import_assertion(
+        (
+            "import sys; import framevitals.pipeline; "
+            "assert 'framevitals.visualizer' not in sys.modules; "
+            "assert 'framevitals.explainability' not in sys.modules; "
+            "assert 'matplotlib' not in sys.modules; "
+            "assert 'seaborn' not in sys.modules"
+        )
+    )
+    assert completed.returncode == 0, completed.stderr
+
+
+def test_explainability_import_does_not_require_plotting_stack():
+    completed = _run_import_assertion(
+        (
+            "import sys; import framevitals.explainability; "
+            "assert 'matplotlib' not in sys.modules; "
+            "assert 'seaborn' not in sys.modules"
+        )
+    )
     assert completed.returncode == 0, completed.stderr
 
 

From bb0d4eeef644aaeecafe6e2b7506ba474b7802d6 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:41:58 +0530
Subject: [PATCH 245/439] test: skip web boundary check without Flask

---
 tests/test_web_dependency_boundaries.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_web_dependency_boundaries.py b/tests/test_web_dependency_boundaries.py
index 3c28f1d..32f13f2 100644
--- a/tests/test_web_dependency_boundaries.py
+++ b/tests/test_web_dependency_boundaries.py
@@ -3,6 +3,10 @@
 import sys
 from pathlib import Path
 
+import pytest
+
+pytest.importorskip("flask")
+
 
 def test_web_app_does_not_eagerly_import_optional_ai_or_report_stack():
     repo_root = Path(__file__).resolve().parents[1]

From aafc942323e9eed3a1d5d3270989fc16646188b9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:42:19 +0530
Subject: [PATCH 246/439] ci: verify optional dependency boundaries

---
 .github/workflows/test.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9d57087..6a7dda2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -269,10 +269,15 @@ jobs:
           python - <<'PY'
           import flask
           import lightgbm
+          import matplotlib
           import ollama
+          import openpyxl
+          import pydantic
           import pyarrow
           import pyod
+          import seaborn
           import shap
+          import xlrd
           import xgboost
           print("Optional dependency smoke test passed")
           PY
@@ -282,7 +287,9 @@ jobs:
           pytest -v \
             tests/test_ai_agent.py \
             tests/test_ml_engine.py \
-            tests/test_model_diagnostics.py
+            tests/test_model_diagnostics.py \
+            tests/test_visualization_engine.py \
+            tests/test_web_dependency_boundaries.py
 
   frontend:
     name: frontend build

From d82642b4ff001f518aef390b15b52e070d0c2800 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:43:25 +0530
Subject: [PATCH 247/439] feat: ship reusable FrameVitals gate action

---
 action.yml | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 140 insertions(+)
 create mode 100644 action.yml

diff --git a/action.yml b/action.yml
new file mode 100644
index 0000000..b7c1015
--- /dev/null
+++ b/action.yml
@@ -0,0 +1,140 @@
+name: FrameVitals Data Quality Gate
+description: Run FrameVitals drift and contract checks as a CI quality gate.
+author: Parth Dongre
+
+branding:
+  icon: activity
+  color: blue
+
+inputs:
+  current:
+    description: Path to the current dataset in the caller workspace.
+    required: true
+  reference:
+    description: Optional reference dataset path for drift checks.
+    required: false
+    default: ""
+  contract:
+    description: Optional FrameVitals contract JSON path for exact validation.
+    required: false
+    default: ""
+  columns:
+    description: Optional comma-separated columns to include in drift checks.
+    required: false
+    default: ""
+  max-columns:
+    description: Maximum shared columns to compare for drift.
+    required: false
+    default: "30"
+  drift-warn-on:
+    description: Drift severity that changes a passing gate to warning.
+    required: false
+    default: moderate
+  drift-fail-on:
+    description: Drift severity that fails the gate.
+    required: false
+    default: severe
+  fail-on-validation-warning:
+    description: Set to true to promote contract warnings to failures.
+    required: false
+    default: "false"
+  output:
+    description: JSON result path written in the caller workspace.
+    required: false
+    default: framevitals-gate.json
+  python-version:
+    description: Python version used to run FrameVitals.
+    required: false
+    default: "3.12"
+
+outputs:
+  status:
+    description: FrameVitals gate status (pass, warn, or fail).
+    value: ${{ steps.gate.outputs.status }}
+  passed:
+    description: Whether the gate avoided a failing verdict.
+    value: ${{ steps.gate.outputs.passed }}
+  result-path:
+    description: Path to the JSON gate result.
+    value: ${{ steps.gate.outputs.result-path }}
+
+runs:
+  using: composite
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: pip
+        cache-dependency-path: ${{ github.action_path }}/pyproject.toml
+
+    - name: Install FrameVitals action runtime
+      shell: bash
+      env:
+        ACTION_PATH: ${{ github.action_path }}
+      run: |
+        python -m pip install --upgrade pip
+        cd "$ACTION_PATH"
+        python -m pip install ".[arrow]"
+
+    - name: Run FrameVitals quality gate
+      id: gate
+      shell: bash
+      env:
+        CURRENT: ${{ inputs.current }}
+        REFERENCE: ${{ inputs.reference }}
+        CONTRACT: ${{ inputs.contract }}
+        COLUMNS: ${{ inputs.columns }}
+        MAX_COLUMNS: ${{ inputs.max-columns }}
+        DRIFT_WARN_ON: ${{ inputs.drift-warn-on }}
+        DRIFT_FAIL_ON: ${{ inputs.drift-fail-on }}
+        FAIL_ON_VALIDATION_WARNING: ${{ inputs.fail-on-validation-warning }}
+        OUTPUT: ${{ inputs.output }}
+      run: |
+        if [[ -z "$REFERENCE" && -z "$CONTRACT" ]]; then
+          echo "FrameVitals gate requires at least one of reference or contract." >&2
+          exit 2
+        fi
+
+        args=(
+          gate "$CURRENT"
+          --max-columns "$MAX_COLUMNS"
+          --drift-warn-on "$DRIFT_WARN_ON"
+          --drift-fail-on "$DRIFT_FAIL_ON"
+          --format json
+          --output "$OUTPUT"
+        )
+
+        if [[ -n "$REFERENCE" ]]; then
+          args+=(--reference "$REFERENCE")
+        fi
+        if [[ -n "$CONTRACT" ]]; then
+          args+=(--contract "$CONTRACT")
+        fi
+        if [[ -n "$COLUMNS" ]]; then
+          args+=(--columns "$COLUMNS")
+        fi
+        if [[ "$FAIL_ON_VALIDATION_WARNING" == "true" ]]; then
+          args+=(--fail-on-validation-warning)
+        fi
+
+        set +e
+        framevitals "${args[@]}"
+        exit_code=$?
+        set -e
+
+        python - "$OUTPUT" "$GITHUB_OUTPUT" <<'PY'
+        import json
+        import sys
+        from pathlib import Path
+
+        result_path = Path(sys.argv[1])
+        github_output = Path(sys.argv[2])
+        payload = json.loads(result_path.read_text(encoding="utf-8"))
+        with github_output.open("a", encoding="utf-8") as handle:
+            handle.write(f"status={payload.get('status', 'unknown')}\n")
+            handle.write(f"passed={str(bool(payload.get('passed'))).lower()}\n")
+            handle.write(f"result-path={result_path}\n")
+        PY
+
+        exit "$exit_code"

From ad309c01285918e179c26ed43df1c7ea8af0331b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:43:40 +0530
Subject: [PATCH 248/439] ci: smoke-test reusable gate action

---
 .github/workflows/action-smoke.yml | 76 ++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)
 create mode 100644 .github/workflows/action-smoke.yml

diff --git a/.github/workflows/action-smoke.yml b/.github/workflows/action-smoke.yml
new file mode 100644
index 0000000..0a44000
--- /dev/null
+++ b/.github/workflows/action-smoke.yml
@@ -0,0 +1,76 @@
+name: Gate Action Smoke Test
+
+on:
+  push:
+    branches: [main, dev, "develop/august"]
+    paths:
+      - action.yml
+      - pyproject.toml
+      - src/framevitals/cli.py
+      - src/framevitals/operations.py
+      - .github/workflows/action-smoke.yml
+  pull_request:
+    paths:
+      - action.yml
+      - pyproject.toml
+      - src/framevitals/cli.py
+      - src/framevitals/operations.py
+      - .github/workflows/action-smoke.yml
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Create stable fixture datasets
+        shell: bash
+        run: |
+          cat > reference.csv <<'CSV'
+          age,income,segment
+          21,30000,basic
+          34,62000,pro
+          48,81000,pro
+          52,90000,enterprise
+          CSV
+          cp reference.csv current.csv
+
+      - name: Run local FrameVitals gate action
+        id: framevitals
+        uses: ./
+        with:
+          current: current.csv
+          reference: reference.csv
+          output: gate-result.json
+
+      - name: Verify action outputs
+        shell: bash
+        env:
+          STATUS: ${{ steps.framevitals.outputs.status }}
+          PASSED: ${{ steps.framevitals.outputs.passed }}
+          RESULT_PATH: ${{ steps.framevitals.outputs.result-path }}
+        run: |
+          test "$STATUS" = "pass"
+          test "$PASSED" = "true"
+          test "$RESULT_PATH" = "gate-result.json"
+          test -s gate-result.json
+          python - <<'PY'
+          import json
+          from pathlib import Path
+
+          payload = json.loads(Path("gate-result.json").read_text(encoding="utf-8"))
+          assert payload["status"] == "pass"
+          assert payload["passed"] is True
+          assert payload["checks_run"] == ["drift"]
+          PY

From bc48a5640dc10cd7390f982867c4fa872f90b5b1 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:44:16 +0530
Subject: [PATCH 249/439] fix: make gate action setup portable

---
 action.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/action.yml b/action.yml
index b7c1015..ab45c5a 100644
--- a/action.yml
+++ b/action.yml
@@ -65,8 +65,6 @@ runs:
       uses: actions/setup-python@v6
       with:
         python-version: ${{ inputs.python-version }}
-        cache: pip
-        cache-dependency-path: ${{ github.action_path }}/pyproject.toml
 
     - name: Install FrameVitals action runtime
       shell: bash

From 0c38044909c76e49ad54f557dbf2fcaf9ad30a6d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:44:52 +0530
Subject: [PATCH 250/439] docs: document contributor architecture boundaries

---
 CONTRIBUTING.md | 72 ++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 69 insertions(+), 3 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 213dde2..3ec9910 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -2,6 +2,8 @@
 
 Thanks for considering a contribution to FrameVitals.
 
+FrameVitals is being built as a package-first data-health and quality-gate library. Contributions are most valuable when they make the public workflow more reliable, source-aware, transparent, and maintainable rather than simply adding more unrelated diagnostics.
+
 ## Development setup
 
 ```bash
@@ -36,11 +38,55 @@ pytest
 4. Open the pull request against `dev`.
 5. Promote tested release changes from `dev` to `main` through a release pull request.
 
-## Source layout
+## Architecture boundaries
 
 `src/framevitals/` is the canonical Python package. New reusable code belongs there and should import through the `framevitals.*` namespace.
 
-The Flask API and React dashboard are optional interfaces around the package. Product logic should stay in `src/framevitals/` rather than being duplicated in application code.
+The main layers have distinct responsibilities:
+
+| Layer | Responsibility |
+| --- | --- |
+| `sources.py` | Normalize inputs and expose metadata, projection, loading, and optional streaming |
+| `focused.py` | Run one requested diagnostic without invoking unrelated pipeline stages |
+| `analysis_api.py` | Dispatch full analysis through the appropriate source-aware execution path |
+| `streaming_pipeline.py` | Orchestrate bounded full analysis for streaming-capable sources |
+| `planning_api.py` | Preview execution decisions without running heavy analyses |
+| `operations.py` | Cleaning, drift, contracts, validation, and quality gates |
+| `result.py` / `quality_results.py` | Dict-compatible public result objects and helpers |
+| `cli.py` | Thin command-line interface over canonical package APIs |
+| `app.py` / `frontend/` | Optional interfaces; they must not duplicate package logic |
+
+A compatibility module may delegate to these layers, but it should not contain a second implementation of the same public operation.
+
+## Source-aware execution
+
+Do not call `source.load()` automatically just because an analysis accepts a file path.
+
+When a `StreamingDatasetSource` can safely support the operation:
+
+1. inspect source metadata first;
+2. derive execution budgets from the **true** source shape;
+3. project only required columns where practical;
+4. stream or sample deterministically within the budget;
+5. expose execution provenance in the returned payload.
+
+Approximate work must be labelled honestly. If a metric comes from a bounded row sample, do not present it as exact. Conversely, checks such as uniqueness, allowed-value validation, or hard numeric bounds must not silently become sampled checks when exactness is part of the public contract.
+
+## Optional dependencies
+
+The base package should not import optional stacks unless their feature is actually requested.
+
+Current capability groups include:
+
+- `framevitals[arrow]` for Arrow-backed streaming;
+- `framevitals[excel]` for XLS/XLSX readers;
+- `framevitals[plot]` for chart/PDF plotting support;
+- `framevitals[ml]` for heavier model/explainability integrations;
+- `framevitals[ai]` for agentic AI integrations;
+- `framevitals[web]` for the Flask runtime;
+- `framevitals[all]` for all optional runtime capabilities.
+
+If a new optional dependency is introduced, prefer a lazy import plus a clear install hint. Add a dependency-boundary test when accidental eager imports would make the base install heavier or break another extra.
 
 ## Tests
 
@@ -50,6 +96,7 @@ Useful checks include:
 pytest
 pytest tests/test_public_api.py
 pytest tests/test_framevitals_pipeline.py
+pytest tests/test_optional_dependency_boundaries.py
 pytest tests/test_package_boundary.py
 python -m compileall src/framevitals app.py
 python -m build
@@ -57,6 +104,15 @@ python -m twine check dist/*
 framevitals --version
 ```
 
+For source-aware changes, also run the relevant focused suites. Examples:
+
+```bash
+pytest tests/test_parquet_streaming.py
+pytest tests/test_csv_streaming.py
+pytest tests/test_streaming_drift.py
+pytest tests/test_statistics_streaming.py
+```
+
 For the optional React dashboard:
 
 ```bash
@@ -65,13 +121,22 @@ npm ci
 npm run build
 ```
 
+Performance-sensitive changes should use the benchmark harness rather than intuition alone:
+
+```bash
+python benchmarks/benchmark_profile_scale.py --rows 50000 --scenarios numpy auto parquet
+```
+
 ## Style
 
 - Prefer clear Python over clever Python.
 - Keep the public API deliberate and small.
 - Avoid hidden global state in reusable library code.
 - Return structured, JSON-friendly values where practical.
+- Preserve dict compatibility for public result objects during the 0.x series unless a breaking change is explicitly planned.
 - Optional analyses should fail gracefully when a dependency is unavailable.
+- Keep source shape, sample shape, and materialization state distinct in execution metadata.
+- Use Rust only for measured hot paths; keep orchestration and public semantics readable in Python.
 - Do not commit generated reports, uploads, cleaned datasets, caches, virtual environments, build output, or frontend compiler artifacts.
 
 ## Pull requests
@@ -82,7 +147,8 @@ A good pull request explains:
 - why the chosen approach is appropriate;
 - what behavior changed;
 - what tests were run;
-- any compatibility or performance implications.
+- any compatibility or performance implications;
+- whether the change affects exactness, sampling, optional dependencies, or public result schemas.
 
 For large features or public-API changes, open an issue first so the design can be discussed before implementation.
 

From a30fe7c31c375d9d174276872d60d01b78b61cca Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:45:49 +0530
Subject: [PATCH 251/439] docs: document extras and reusable gate action

---
 README.md | 58 +++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 48 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 964518d..f234f17 100644
--- a/README.md
+++ b/README.md
@@ -80,17 +80,19 @@ FrameVitals supports **Python 3.11, 3.12, and 3.13**. The current public release
 pip install framevitals
 ```
 
-Optional capabilities are installed as extras:
+Optional capabilities are installed as extras so the base data-health engine does not need to import every integration stack:
 
 ```bash
 pip install "framevitals[arrow]"  # bounded Parquet / compatible CSV / TSV streaming
+pip install "framevitals[excel]"  # XLS/XLSX readers
+pip install "framevitals[plot]"   # Matplotlib/Seaborn chart and PDF plotting support
 pip install "framevitals[ml]"     # XGBoost, LightGBM, PyOD, SHAP
-pip install "framevitals[ai]"     # Ollama-backed AI features
+pip install "framevitals[ai]"     # Ollama-backed agentic AI features
 pip install "framevitals[web]"    # Flask web runtime
 pip install "framevitals[all]"    # all optional runtime capabilities
 ```
 
-Parquet streaming requires the `arrow` extra. With Arrow installed, compatible CSV and TSV inputs can also use the source-aware streaming engine; without Arrow they retain the normal pandas-backed file path.
+Parquet streaming requires the `arrow` extra. With Arrow installed, compatible CSV and TSV inputs can also use the source-aware streaming engine; without Arrow they retain the normal pandas-backed file path. Excel readers are optional through `excel`. Plotting is optional through `plot`, and structured explainability can still fall back to non-plotting importance results when chart support is unavailable.
 
 ## Quick start
 
@@ -206,6 +208,27 @@ fv.gate(current, reference=reference, contract=contract)
 
 Contract validation remains exact. Drift can use bounded source-aware sampling where a streaming input supports it, and the returned `execution` block records that distinction.
 
+### GitHub Actions
+
+The repository also ships a reusable composite action at the repository root. Once using a release/tag that contains the action, a workflow can gate a dataset without writing custom installation or exit-code glue:
+
+```yaml
+- uses: parthdongre/FrameVitals@main
+  id: framevitals
+  with:
+    current: data/production.parquet
+    reference: data/training.parquet
+    contract: data/contract.json
+    drift-warn-on: moderate
+    drift-fail-on: severe
+    output: framevitals-gate.json
+
+- name: Show verdict
+  run: echo "FrameVitals status: ${{ steps.framevitals.outputs.status }}"
+```
+
+For production workflows, pin the action to a released tag or commit rather than a moving branch. The action installs the matching FrameVitals source plus the Arrow capability and exposes `status`, `passed`, and `result-path` outputs.
+
 ## Snapshots
 
 An `AnalysisResult` can be reduced to a compact versioned monitoring snapshot:
@@ -413,21 +436,33 @@ report.snapshot("snapshot.json")
 
 Drift, validation and gate workflows similarly return dict-compatible `DriftResult`, `ValidationResult` and `GateResult` objects with convenience properties while preserving JSON-friendly mapping behavior during the `0.x` series.
 
-## Optional ML and AI features
+## Optional capabilities
 
-The default package contains the core data-health engine. Additional integrations are separated into extras.
+The default package contains the core data-health engine. Additional integrations are separated into extras and are imported only when their feature is requested.
 
 ```bash
 pip install "framevitals[ml]"
 ```
 
-Adds integrations including XGBoost, LightGBM, PyOD and SHAP.
+Adds heavier model integrations including XGBoost, LightGBM, PyOD and SHAP.
 
 ```bash
 pip install "framevitals[ai]"
 ```
 
-Adds Ollama-backed interpretation and question-answering features. AI is treated as an optional explanation layer; computed diagnostics remain usable without a reachable model.
+Adds Ollama-backed agentic interpretation and question-answering features. AI remains an optional explanation layer; computed diagnostics remain usable without it.
+
+```bash
+pip install "framevitals[plot]"
+```
+
+Adds Matplotlib/Seaborn-backed chart rendering and PDF/report plotting. Structured diagnostics do not require this extra.
+
+```bash
+pip install "framevitals[excel]"
+```
+
+Adds XLS/XLSX reader engines. CSV/TSV/JSON workflows remain available without them.
 
 ## Web dashboard
 
@@ -446,6 +481,8 @@ npm ci
 npm run dev
 ```
 
+The `web` extra contains the server runtime only. Install `plot` as well for server-side chart/PDF artifacts and `ai` for the agentic Q&A path. Missing optional AI/report capabilities are loaded lazily instead of preventing the web server from starting.
+
 Typical local endpoints:
 
 - Flask API: `http://127.0.0.1:5055`
@@ -465,7 +502,7 @@ FrameVitals is being built around a few constraints that are easy to lose in ana
 - **Exact where correctness requires it** — contract constraints are not silently weakened to sampled checks.
 - **Safe defaults** — no unexpected artifact writes and graceful optional-feature fallbacks.
 - **Small workflow surface** — make analyze → compare → validate → gate obvious while keeping focused diagnostics available.
-- **Optional integrations** — ML, AI, Arrow and web capabilities can be installed independently.
+- **Optional integrations** — Arrow, Excel, plotting, ML, AI, and web capabilities can be installed independently.
 
 ## Project layout
 
@@ -478,9 +515,10 @@ FrameVitals is being built around a few constraints that are easy to lose in ana
 ├── frontend/                 # optional React + TypeScript dashboard
 ├── templates/                # Flask report pages
 ├── static/                   # web/report assets
+├── action.yml                # reusable GitHub Actions quality gate
 ├── app.py                    # optional Flask API/server
 ├── pyproject.toml            # package metadata and dependency groups
-└── .github/workflows/        # CI, package validation and publishing
+└── .github/workflows/        # CI, package, action, benchmark and publishing workflows
 ```
 
 New reusable Python code belongs in `src/framevitals/` and should import through the `framevitals.*` namespace.
@@ -508,7 +546,7 @@ On Windows PowerShell:
 .venv\Scripts\Activate.ps1
 ```
 
-CI validates the core package across Python 3.11–3.13, Arrow streaming paths, optional features, the Rust workspace/native bridge, React build, wheel contents, distribution metadata, and a clean-wheel install.
+CI validates the core package across Python 3.11–3.13, Arrow streaming paths, optional dependency boundaries, the reusable Gate Action, the Rust workspace/native bridge, React build, wheel contents, distribution metadata, release-version consistency, and a clean-wheel install. A separate benchmark workflow records reproducible profile time and peak-RSS measurements on `main` or when manually dispatched.
 
 Development is integrated through `dev`; `main` is kept release-ready.
 

From 183d766cc174de4fbf1880de947e4b96358ea906 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:46:12 +0530
Subject: [PATCH 252/439] docs: record optional stack and gate action overhaul

---
 CHANGELOG.md | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fdb8a76..13fdb40 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,8 @@ Development continues on the development branches. Changes intended for the next
 - Structured exact contract validation through `framevitals.validate()`
 - `framevitals.gate()` for one CI-friendly pass/warn/fail verdict combining validation and drift
 - `framevitals gate` CLI command with configurable drift warning/failure thresholds, optional contract enforcement, JSON output, and CI exit codes
+- Reusable root `action.yml` GitHub Action for running FrameVitals quality gates with status/result outputs
+- Path-scoped smoke CI for the reusable Gate Action
 - `framevitals plan` for previewing applicable modules, execution budgets, and source planning without running the heavy pipeline
 - Compact versioned analysis snapshots with deterministic fingerprints, JSON persistence, and snapshot-to-snapshot schema/health/ML-readiness/finding diffs
 - Dict-compatible result objects including `AnalysisResult`, `DriftResult`, `ValidationResult`, `GateResult`, and per-column result views
@@ -24,14 +26,17 @@ Development continues on the development branches. Changes intended for the next
 - Execution provenance that distinguishes exact full-stream metrics, bounded row samples, estimates, and full materialization
 - Optional Rust native core and Python bridge for accelerated streaming numeric state, cardinality/quantile sketches, and backend routing
 - Dedicated Arrow/NumPy-fallback and Rust/native CI coverage
-- Benchmark harnesses for scale-sensitive profiling behavior
+- Reproducible performance benchmark workflow with time/RSS summaries and retained JSON artifacts
 - PEP 561 `py.typed` marker for downstream type-checking support
 - Optional `excel` dependency group for XLS/XLSX readers
+- Optional `plot` dependency group for Matplotlib/Seaborn chart and PDF/report support
+- Contributor architecture guidance covering canonical layers, streaming provenance, optional-dependency boundaries, and performance-sensitive changes
 
 ### Changed
 
 - Full `framevitals.analyze()` now dispatches streaming-capable sources through a bounded source-aware pipeline when artifacts do not require materialization
 - Drift comparison now preserves exact source shape metadata while bounding value-distribution work for streaming sources
+- Flask `/api/compare` now routes through the same canonical source-aware drift engine as Python and CLI callers
 - `framevitals.gate()` reuses the same canonical drift and validation engines instead of maintaining separate data-loading logic
 - Contract validation explicitly remains exact for uniqueness, allowed-value, nullability, and bound constraints rather than silently weakening them to sampled checks
 - `framevitals.api` is now a lazy compatibility facade over the canonical focused, analysis, planning, and operations engines instead of a second eager implementation
@@ -39,9 +44,14 @@ Development continues on the development branches. Changes intended for the next
 - CLI `--version` works from a no-dependencies wheel smoke install without importing the analytics stack
 - The `all` extra now actually includes the Arrow capability
 - Excel engines moved out of the default dependency set into `framevitals[excel]`; AI-only Pydantic moved into `framevitals[ai]`
+- Matplotlib and Seaborn moved out of the default dependency set into `framevitals[plot]`
+- Explainability no longer requires Matplotlib at module import time; structured importance can run without plotting and SHAP chart rendering is optional
+- Flask startup no longer eagerly imports the agentic AI or PDF/report stacks; those capabilities load only when their endpoints are used
 - Excel loading now reports a FrameVitals-specific install hint when the optional reader capability is missing
-- README and roadmap now describe the source-aware analyze → compare → validate → gate → monitor direction and document actual CLI exit semantics
+- README and roadmap now describe the source-aware analyze → compare → validate → gate → monitor direction, optional capability groups, reusable Gate Action, and actual CLI exit semantics
 - Package CI now requires the `py.typed` marker to be present in built wheels
+- Test and package workflows now use read-only default permissions and cancel stale runs on the same branch
+- Release publishing now verifies that the GitHub release tag, `pyproject.toml` version, and `framevitals.__version__` match before building/publishing
 
 ### Fixed
 
@@ -50,6 +60,8 @@ Development continues on the development branches. Changes intended for the next
 - Streaming column-role cardinality metadata distinguishes bounded-sample cardinality from full-stream native estimates
 - CSV/TSV streaming sources preserve `FileSource` compatibility for existing callers and tests
 - Dedicated Arrow CI now exercises streaming statistics, drift, CSV/TSV sources, and the quality-gate CLI
+- Web runtime dependency boundaries no longer force AI/report/plot packages to import during Flask startup
+- Reusable Gate Action setup no longer relies on an invalid absolute `setup-python` cache dependency path
 
 ## 0.1.0 - 2026-08-15
 

From 30a96b806f3953ad173a769abb17461fbdc341bd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:46:55 +0530
Subject: [PATCH 253/439] feat: add persistent snapshot history API

---
 src/framevitals/snapshots.py | 126 +++++++++++++++++++++++++++++++++--
 1 file changed, 122 insertions(+), 4 deletions(-)

diff --git a/src/framevitals/snapshots.py b/src/framevitals/snapshots.py
index 8892919..d22ee67 100644
--- a/src/framevitals/snapshots.py
+++ b/src/framevitals/snapshots.py
@@ -1,9 +1,10 @@
-"""Compact versioned snapshots derived from FrameVitals analysis results."""
+"""Compact versioned snapshots and lightweight local monitoring history."""
 
 from __future__ import annotations
 
 import hashlib
 import json
+import re
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Mapping
@@ -72,6 +73,26 @@ def _fingerprint(state: Mapping[str, Any]) -> str:
     return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
 
 
+def _created_at(snapshot: Mapping[str, Any]) -> datetime:
+    value = snapshot.get("created_at")
+    if not isinstance(value, str):
+        raise ValueError("Snapshot is missing a valid created_at timestamp.")
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError as exc:
+        raise ValueError(f"Snapshot has an invalid created_at timestamp: {value!r}") from exc
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed.astimezone(timezone.utc)
+
+
+def _safe_label(value: str | None) -> str | None:
+    if value is None:
+        return None
+    cleaned = re.sub(r"[^A-Za-z0-9._-]+", "-", value.strip()).strip("-._")
+    return cleaned[:80] or None
+
+
 class AnalysisSnapshot(dict):
     """Small JSON-friendly state record for monitoring and history."""
 
@@ -133,6 +154,7 @@ def load_snapshot(path: str | Path) -> AnalysisSnapshot:
         )
     if not isinstance(payload.get("state"), dict):
         raise ValueError("Snapshot is missing a valid state object.")
+    _created_at(payload)
     return AnalysisSnapshot(payload)
 
 
@@ -179,9 +201,7 @@ def compare_snapshots(
     ref_findings = set(ref_state.get("finding_codes", []) or [])
     cur_findings = set(cur_state.get("finding_codes", []) or [])
 
-    changed = bool(
-        reference.get("fingerprint") != current.get("fingerprint")
-    )
+    changed = bool(reference.get("fingerprint") != current.get("fingerprint"))
     return {
         "changed": changed,
         "reference_fingerprint": reference.get("fingerprint"),
@@ -207,3 +227,101 @@ def compare_snapshots(
             "resolved": sorted(ref_findings - cur_findings),
         },
     }
+
+
+class SnapshotHistory:
+    """Filesystem-backed history of compact FrameVitals snapshots.
+
+    The history store never writes raw datasets. It persists only the compact
+    snapshot representation and provides lightweight timeline/latest/diff
+    helpers for local monitoring and CI workflows.
+    """
+
+    def __init__(self, directory: str | Path = ".framevitals/history") -> None:
+        self.directory = Path(directory)
+
+    def __len__(self) -> int:
+        return len(self.paths())
+
+    def paths(self) -> list[Path]:
+        """Return snapshot files in chronological filename order."""
+        if not self.directory.exists():
+            return []
+        return sorted(path for path in self.directory.glob("*.json") if path.is_file())
+
+    def snapshots(self) -> list[AnalysisSnapshot]:
+        """Load all valid snapshots ordered by their created-at timestamp."""
+        items = [load_snapshot(path) for path in self.paths()]
+        items.sort(key=_created_at)
+        return items
+
+    def add(
+        self,
+        result_or_snapshot: Mapping[str, Any],
+        *,
+        label: str | None = None,
+    ) -> Path:
+        """Persist an analysis result or an existing snapshot and return its path."""
+        if result_or_snapshot.get("snapshot_schema_version") is not None:
+            if result_or_snapshot.get("snapshot_schema_version") != SNAPSHOT_SCHEMA_VERSION:
+                raise ValueError(
+                    "Unsupported FrameVitals snapshot schema version: "
+                    f"{result_or_snapshot.get('snapshot_schema_version')!r}"
+                )
+            if not isinstance(result_or_snapshot.get("state"), Mapping):
+                raise ValueError("Snapshot is missing a valid state object.")
+            snapshot = AnalysisSnapshot(dict(result_or_snapshot))
+            created_at = _created_at(snapshot)
+        else:
+            snapshot = create_snapshot(result_or_snapshot)
+            created_at = _created_at(snapshot)
+
+        self.directory.mkdir(parents=True, exist_ok=True)
+        timestamp = created_at.strftime("%Y%m%dT%H%M%S.%fZ")
+        fingerprint = str(snapshot.get("fingerprint") or "unknown")[:12]
+        label_part = _safe_label(label)
+        filename = "_".join(
+            part for part in (timestamp, label_part, fingerprint) if part
+        ) + ".json"
+        path = self.directory / filename
+        snapshot.to_json(path)
+        return path
+
+    def latest(self) -> AnalysisSnapshot | None:
+        """Return the newest snapshot or ``None`` for an empty history."""
+        items = self.snapshots()
+        return items[-1] if items else None
+
+    def previous(self) -> AnalysisSnapshot | None:
+        """Return the snapshot immediately before the latest one, if available."""
+        items = self.snapshots()
+        return items[-2] if len(items) >= 2 else None
+
+    def compare_latest(self) -> dict[str, Any]:
+        """Compare the two newest snapshots."""
+        items = self.snapshots()
+        if len(items) < 2:
+            raise ValueError("Snapshot history needs at least two entries to compare.")
+        return compare_snapshots(items[-2], items[-1])
+
+    def timeline(self) -> list[dict[str, Any]]:
+        """Return compact chronological monitoring points for charts or logs."""
+        rows: list[dict[str, Any]] = []
+        for snapshot in self.snapshots():
+            state = _as_mapping(snapshot.get("state"))
+            dataset = _as_mapping(state.get("dataset"))
+            health = _as_mapping(state.get("health"))
+            ml = _as_mapping(state.get("ml_readiness"))
+            findings = state.get("finding_codes", [])
+            if not isinstance(findings, list):
+                findings = []
+            rows.append({
+                "created_at": snapshot.get("created_at"),
+                "fingerprint": snapshot.get("fingerprint"),
+                "filename": _as_mapping(snapshot.get("source")).get("filename"),
+                "shape": dict(_as_mapping(dataset.get("shape"))),
+                "health_score": _number(health.get("overall_score")),
+                "ml_readiness_score": _number(ml.get("score")),
+                "finding_count": len(findings),
+            })
+        return rows

From 2f5ec9ac6fa808074bb32ea40c344cb557b0cfa9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:47:17 +0530
Subject: [PATCH 254/439] feat: expose snapshot history publicly

---
 src/framevitals/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index c2182b0..bbfad8f 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -10,6 +10,7 @@
 from framevitals.result import AnalysisResult, ColumnResult
 from framevitals.snapshots import (
     AnalysisSnapshot,
+    SnapshotHistory,
     compare_snapshots,
     create_snapshot,
     load_snapshot,
@@ -290,6 +291,7 @@ def gate(
     "AnalysisPlan",
     "AnalysisResult",
     "AnalysisSnapshot",
+    "SnapshotHistory",
     "CleaningPlan",
     "ColumnResult",
     "DriftResult",

From f376d8e4c596fe8e938bfe925f76ab02321830ee Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:47:46 +0530
Subject: [PATCH 255/439] test: cover persistent snapshot history

---
 tests/test_snapshots.py | 58 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 57 insertions(+), 1 deletion(-)

diff --git a/tests/test_snapshots.py b/tests/test_snapshots.py
index 7a0c39b..84e58f9 100644
--- a/tests/test_snapshots.py
+++ b/tests/test_snapshots.py
@@ -1,7 +1,14 @@
 import json
 
+import pytest
+
 from framevitals.result import AnalysisResult
-from framevitals.snapshots import compare_snapshots, load_snapshot
+from framevitals.snapshots import (
+    SnapshotHistory,
+    compare_snapshots,
+    create_snapshot,
+    load_snapshot,
+)
 
 
 def _result(*, columns=None, health=80.0, missing=None, findings=None):
@@ -89,3 +96,52 @@ def test_snapshot_json_roundtrip(tmp_path):
     assert payload["fingerprint"] == snapshot["fingerprint"]
     loaded = load_snapshot(path)
     assert loaded.diff(snapshot)["changed"] is False
+
+
+def test_snapshot_history_persists_orders_and_compares_latest(tmp_path):
+    history = SnapshotHistory(tmp_path / "history")
+    baseline = create_snapshot(_result(health=90.0))
+    baseline["created_at"] = "2026-08-14T08:00:00+00:00"
+    current = create_snapshot(
+        _result(
+            health=72.5,
+            findings=[{"code": "quality.high_missingness.age"}],
+        )
+    )
+    current["created_at"] = "2026-08-15T08:00:00+00:00"
+
+    first_path = history.add(baseline, label="baseline release")
+    second_path = history.add(current, label="production")
+
+    assert len(history) == 2
+    assert "baseline-release" in first_path.name
+    assert "production" in second_path.name
+    assert history.previous()["fingerprint"] == baseline["fingerprint"]
+    assert history.latest()["fingerprint"] == current["fingerprint"]
+
+    diff = history.compare_latest()
+    assert diff["changed"] is True
+    assert diff["health_delta"] == -17.5
+    assert diff["findings"]["new"] == ["quality.high_missingness.age"]
+
+    timeline = history.timeline()
+    assert [row["health_score"] for row in timeline] == [90.0, 72.5]
+    assert [row["finding_count"] for row in timeline] == [0, 1]
+    assert all(row["filename"] == "customers.csv" for row in timeline)
+
+
+def test_snapshot_history_can_add_analysis_result_directly(tmp_path):
+    history = SnapshotHistory(tmp_path / "history")
+    path = history.add(_result(health=88.0), label="nightly")
+
+    assert path.exists()
+    assert len(history) == 1
+    assert history.latest()["state"]["health"]["overall_score"] == 88.0
+
+
+def test_snapshot_history_requires_two_entries_for_latest_diff(tmp_path):
+    history = SnapshotHistory(tmp_path / "history")
+    history.add(_result())
+
+    with pytest.raises(ValueError, match="at least two"):
+        history.compare_latest()

From 904650ebc879434b3119cc186f7e8abea48026c8 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:48:33 +0530
Subject: [PATCH 256/439] feat: add extensible custom data checks

---
 src/framevitals/checks.py | 206 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 206 insertions(+)
 create mode 100644 src/framevitals/checks.py

diff --git a/src/framevitals/checks.py b/src/framevitals/checks.py
new file mode 100644
index 0000000..d3c2bb8
--- /dev/null
+++ b/src/framevitals/checks.py
@@ -0,0 +1,206 @@
+"""Extensible user-defined checks for FrameVitals quality gates."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+import pandas as pd
+
+from framevitals.sources import resolve_source
+
+
+CheckSeverity = Literal["warning", "error"]
+CheckFunction = Callable[[pd.DataFrame], bool | Mapping[str, Any]]
+DataInput = str | Path | pd.DataFrame
+
+
+def _slug(value: str) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_")
+    return slug or "check"
+
+
+def _validate_severity(value: str) -> CheckSeverity:
+    if value not in {"warning", "error"}:
+        raise ValueError("check severity must be 'warning' or 'error'.")
+    return value  # type: ignore[return-value]
+
+
+@dataclass(frozen=True, slots=True)
+class DataCheck:
+    """Named user-defined data check with gate severity metadata."""
+
+    name: str
+    function: CheckFunction
+    severity: CheckSeverity = "error"
+    description: str | None = None
+
+    def __post_init__(self) -> None:
+        if not self.name.strip():
+            raise ValueError("check name must not be empty.")
+        if not callable(self.function):
+            raise TypeError("check function must be callable.")
+        _validate_severity(self.severity)
+
+    def __call__(self, dataframe: pd.DataFrame) -> bool | Mapping[str, Any]:
+        return self.function(dataframe)
+
+
+def check(
+    name: str | None = None,
+    *,
+    severity: CheckSeverity = "error",
+    description: str | None = None,
+):
+    """Decorate a DataFrame predicate as a reusable :class:`DataCheck`.
+
+    A check can return a boolean or a mapping containing at least ``passed``.
+    Optional mapping keys include ``message`` and ``details``.
+    """
+    resolved_severity = _validate_severity(severity)
+
+    def decorator(function: CheckFunction) -> DataCheck:
+        resolved_name = (name or getattr(function, "__name__", "check")).strip()
+        return DataCheck(
+            name=resolved_name,
+            function=function,
+            severity=resolved_severity,
+            description=description,
+        )
+
+    return decorator
+
+
+def _normalize_check(value: DataCheck | CheckFunction) -> DataCheck:
+    if isinstance(value, DataCheck):
+        return value
+    if not callable(value):
+        raise TypeError("custom checks must be DataCheck instances or callables.")
+    return DataCheck(
+        name=getattr(value, "__name__", "check"),
+        function=value,
+    )
+
+
+def _normalize_outcome(
+    definition: DataCheck,
+    raw: bool | Mapping[str, Any],
+) -> dict[str, Any]:
+    if isinstance(raw, bool):
+        passed = raw
+        message = (
+            f"{definition.name} passed."
+            if passed
+            else f"{definition.name} failed."
+        )
+        details: Any = None
+    elif isinstance(raw, Mapping):
+        if "passed" not in raw:
+            raise ValueError(
+                f"Custom check {definition.name!r} returned a mapping without 'passed'."
+            )
+        passed = bool(raw["passed"])
+        default_message = (
+            f"{definition.name} passed."
+            if passed
+            else f"{definition.name} failed."
+        )
+        message = str(raw.get("message") or default_message)
+        details = raw.get("details")
+    else:
+        raise TypeError(
+            f"Custom check {definition.name!r} must return bool or a mapping."
+        )
+
+    return {
+        "name": definition.name,
+        "code": f"custom.{_slug(definition.name)}",
+        "passed": passed,
+        "severity": definition.severity,
+        "description": definition.description,
+        "message": message,
+        "details": details,
+    }
+
+
+def run_checks(
+    data: DataInput,
+    checks: Sequence[DataCheck | CheckFunction],
+) -> dict[str, Any]:
+    """Run user-defined checks exactly against a materialized DataFrame.
+
+    Custom Python callables can inspect arbitrary row-level relationships, so
+    FrameVitals does not silently sample their input. File-backed inputs are
+    materialized intentionally and that decision is disclosed in ``execution``.
+    """
+    definitions = [_normalize_check(value) for value in checks]
+    if not definitions:
+        raise ValueError("run_checks requires at least one custom check.")
+
+    source = resolve_source(data)
+    metadata = source.inspect()
+    dataframe = source.load()
+
+    results: list[dict[str, Any]] = []
+    for definition in definitions:
+        try:
+            raw = definition(dataframe.copy())
+            outcome = _normalize_outcome(definition, raw)
+            outcome["execution_error"] = None
+        except Exception as exc:  # user-defined check failures become structured results
+            outcome = {
+                "name": definition.name,
+                "code": f"custom.{_slug(definition.name)}",
+                "passed": False,
+                "severity": "error",
+                "description": definition.description,
+                "message": f"Custom check raised {type(exc).__name__}: {exc}",
+                "details": None,
+                "execution_error": f"{type(exc).__name__}: {exc}",
+            }
+        results.append(outcome)
+
+    failures = [result for result in results if not result["passed"]]
+    error_failures = [
+        result for result in failures if result.get("severity") == "error"
+    ]
+    warning_failures = [
+        result for result in failures if result.get("severity") == "warning"
+    ]
+    status = "fail" if error_failures else "warn" if warning_failures else "pass"
+
+    findings = [
+        {
+            "code": result["code"],
+            "severity": result["severity"],
+            "title": result["name"],
+            "message": result["message"],
+            "details": result.get("details"),
+        }
+        for result in failures
+    ]
+
+    return {
+        "status": status,
+        "passed": status != "fail",
+        "results": results,
+        "findings": findings,
+        "summary": {
+            "checks": len(results),
+            "passed": len(results) - len(failures),
+            "warnings": len(warning_failures),
+            "errors": len(error_failures),
+        },
+        "execution": {
+            "method": "exact_custom_checks",
+            "full_materialization": bool(metadata.kind == "file"),
+            "source": metadata.to_dict(),
+            "reason": (
+                "Arbitrary custom Python checks run on the complete DataFrame; "
+                "FrameVitals does not silently sample user-defined invariants."
+            ),
+        },
+    }

From 4b662680f2048bbe69659ff4443fb1b1bcfeb24c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:50:27 +0530
Subject: [PATCH 257/439] feat: run custom checks inside quality gates

---
 src/framevitals/operations.py | 34 ++++++++++++++++++++++++++++++----
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/src/framevitals/operations.py b/src/framevitals/operations.py
index 67ff8c9..a24da05 100644
--- a/src/framevitals/operations.py
+++ b/src/framevitals/operations.py
@@ -7,7 +7,7 @@
 
 from __future__ import annotations
 
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from pathlib import Path
 from typing import Any
 
@@ -246,15 +246,18 @@ def gate(
     *,
     reference: DataInput | None = None,
     contract: Mapping[str, Any] | None = None,
+    custom_checks: Sequence[Any] | None = None,
     columns: list[str] | None = None,
     max_columns: int = 30,
     drift_warn_on: str = "moderate",
     drift_fail_on: str = "severe",
     fail_on_validation_warning: bool = False,
 ) -> GateResult:
-    """Combine exact contract checks and bounded drift into one CI-friendly verdict."""
-    if reference is None and contract is None:
-        raise ValueError("gate requires at least one of reference= or contract=.")
+    """Combine contracts, custom invariants, and bounded drift into one verdict."""
+    if reference is None and contract is None and not custom_checks:
+        raise ValueError(
+            "gate requires at least one of reference=, contract=, or custom_checks=."
+        )
     if drift_warn_on not in _DRIFT_SEVERITY_RANK:
         raise ValueError("drift_warn_on must be one of: stable, minor, moderate, severe.")
     if drift_fail_on not in _DRIFT_SEVERITY_RANK:
@@ -341,6 +344,24 @@ def fail() -> None:
                     if text and text not in reasons:
                         reasons.append(text)
 
+    if custom_checks:
+        from framevitals.checks import run_checks
+
+        custom_payload = run_checks(current, custom_checks)
+        checks["custom"] = custom_payload
+        custom_status = custom_payload.get("status")
+        if custom_status == "fail":
+            fail()
+        elif custom_status == "warn":
+            warn()
+
+        for result in custom_payload.get("results", [])[:10]:
+            if not isinstance(result, Mapping) or result.get("passed"):
+                continue
+            message = str(result.get("message") or result.get("name") or "Custom check failed.")
+            if message and message not in reasons:
+                reasons.append(message)
+
     return GateResult({
         "status": status,
         "passed": status != "fail",
@@ -364,5 +385,10 @@ def fail() -> None:
                 if isinstance(checks.get("drift"), Mapping)
                 else None
             ),
+            "custom": (
+                checks.get("custom", {}).get("execution")
+                if isinstance(checks.get("custom"), Mapping)
+                else None
+            ),
         },
     })

From ab5a0637b1420873af2bb3e4697a5843d93a3617 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:50:54 +0530
Subject: [PATCH 258/439] feat: expose custom checks through public API

---
 src/framevitals/__init__.py | 35 ++++++++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index bbfad8f..7ca4e08 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -17,6 +17,7 @@
 )
 
 if TYPE_CHECKING:
+    from framevitals.checks import DataCheck
     from framevitals.cleaning_plan import CleaningPlan
 
 __version__ = "0.1.0"
@@ -28,6 +29,10 @@ def __getattr__(name: str):
         from framevitals.cleaning_plan import CleaningPlan
 
         return CleaningPlan
+    if name == "DataCheck":
+        from framevitals.checks import DataCheck
+
+        return DataCheck
     raise AttributeError(name)
 
 
@@ -260,24 +265,49 @@ def validate(data: Any, contract: dict[str, Any]) -> ValidationResult:
     return _validate(data, contract)
 
 
+def check(
+    name: str | None = None,
+    *,
+    severity: str = "error",
+    description: str | None = None,
+):
+    """Decorate a DataFrame predicate as a reusable custom data check."""
+    from framevitals.checks import check as _check
+
+    return _check(
+        name,
+        severity=severity,
+        description=description,
+    )
+
+
+def run_checks(data: Any, checks: Any) -> dict[str, Any]:
+    """Run custom data checks exactly against a dataset."""
+    from framevitals.checks import run_checks as _run_checks
+
+    return _run_checks(data, checks)
+
+
 def gate(
     current: Any,
     *,
     reference: Any = None,
     contract: Any = None,
+    custom_checks: Any = None,
     columns: list[str] | None = None,
     max_columns: int = 30,
     drift_warn_on: str = "moderate",
     drift_fail_on: str = "severe",
     fail_on_validation_warning: bool = False,
 ) -> GateResult:
-    """Run contract/drift checks and return one CI-friendly quality verdict."""
+    """Run contract, custom, and drift checks as one CI-friendly quality verdict."""
     from framevitals.operations import gate as _gate
 
     return _gate(
         current,
         reference=reference,
         contract=contract,
+        custom_checks=custom_checks,
         columns=columns,
         max_columns=max_columns,
         drift_warn_on=drift_warn_on,
@@ -294,6 +324,7 @@ def gate(
     "SnapshotHistory",
     "CleaningPlan",
     "ColumnResult",
+    "DataCheck",
     "DriftResult",
     "GateResult",
     "ValidationResult",
@@ -314,6 +345,8 @@ def gate(
     "compare",
     "infer_contract",
     "validate",
+    "check",
+    "run_checks",
     "gate",
     "available_modules",
     "create_snapshot",

From a09bae34d3bf2d419ddf0b0bc45525046d19bd3d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:51:21 +0530
Subject: [PATCH 259/439] feat: expose custom checks through compatibility API

---
 src/framevitals/api.py | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index cbe0bb1..6014e99 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -250,24 +250,45 @@ def validate(data: DataInput, contract: Mapping[str, Any]) -> ValidationResult:
     return _validate(data, contract)
 
 
+def check(
+    name: str | None = None,
+    *,
+    severity: str = "error",
+    description: str | None = None,
+):
+    """Decorate a predicate as a reusable custom data check."""
+    from framevitals.checks import check as _check
+
+    return _check(name, severity=severity, description=description)
+
+
+def run_checks(data: DataInput, checks: Any) -> dict[str, Any]:
+    """Run exact custom checks through the canonical check engine."""
+    from framevitals.checks import run_checks as _run_checks
+
+    return _run_checks(data, checks)
+
+
 def gate(
     current: DataInput,
     *,
     reference: DataInput | None = None,
     contract: Mapping[str, Any] | None = None,
+    custom_checks: Any = None,
     columns: list[str] | None = None,
     max_columns: int = 30,
     drift_warn_on: str = "moderate",
     drift_fail_on: str = "severe",
     fail_on_validation_warning: bool = False,
 ) -> GateResult:
-    """Run the canonical CI-friendly contract/drift quality gate."""
+    """Run the canonical contract/custom/drift quality gate."""
     from framevitals.operations import gate as _gate
 
     return _gate(
         current,
         reference=reference,
         contract=contract,
+        custom_checks=custom_checks,
         columns=columns,
         max_columns=max_columns,
         drift_warn_on=drift_warn_on,
@@ -293,5 +314,7 @@ def gate(
     "compare",
     "infer_contract",
     "validate",
+    "check",
+    "run_checks",
     "gate",
 ]

From f50dbf342e1965b17853c57c877325a2b9ce667a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:51:43 +0530
Subject: [PATCH 260/439] test: cover extensible custom checks and gate
 integration

---
 tests/test_custom_checks.py | 172 ++++++++++++++++++++++++++++++++++++
 1 file changed, 172 insertions(+)
 create mode 100644 tests/test_custom_checks.py

diff --git a/tests/test_custom_checks.py b/tests/test_custom_checks.py
new file mode 100644
index 0000000..9bbdf3e
--- /dev/null
+++ b/tests/test_custom_checks.py
@@ -0,0 +1,172 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+import framevitals as fv
+
+
+def test_top_level_import_keeps_custom_check_engine_lazy():
+    src_root = Path(__file__).resolve().parents[1] / "src"
+    env = os.environ.copy()
+    env["PYTHONPATH"] = str(src_root)
+    completed = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            "import sys; import framevitals; assert 'framevitals.checks' not in sys.modules",
+        ],
+        cwd=src_root.parent,
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert completed.returncode == 0, completed.stderr
+
+
+def test_check_decorator_and_exact_check_results():
+    frame = pd.DataFrame({"revenue": [10.0, 20.0, 30.0]})
+
+    @fv.check("positive revenue", description="Revenue cannot be negative.")
+    def positive_revenue(df):
+        passed = bool((df["revenue"] >= 0).all())
+        return {
+            "passed": passed,
+            "message": "Revenue is non-negative." if passed else "Negative revenue found.",
+            "details": {"minimum": float(df["revenue"].min())},
+        }
+
+    assert positive_revenue.name == "positive revenue"
+    result = fv.run_checks(frame, [positive_revenue])
+
+    assert result["status"] == "pass"
+    assert result["passed"] is True
+    assert result["summary"] == {"checks": 1, "passed": 1, "warnings": 0, "errors": 0}
+    assert result["results"][0]["code"] == "custom.positive_revenue"
+    assert result["results"][0]["details"] == {"minimum": 10.0}
+    assert result["execution"]["method"] == "exact_custom_checks"
+    assert result["execution"]["full_materialization"] is False
+
+
+def test_warning_and_error_checks_produce_distinct_statuses():
+    frame = pd.DataFrame({"value": [1, 2, 3]})
+
+    @fv.check("soft expectation", severity="warning")
+    def soft_expectation(df):
+        return False
+
+    @fv.check("hard expectation", severity="error")
+    def hard_expectation(df):
+        return {"passed": False, "message": "Hard invariant failed."}
+
+    warning_only = fv.run_checks(frame, [soft_expectation])
+    assert warning_only["status"] == "warn"
+    assert warning_only["passed"] is True
+    assert warning_only["summary"]["warnings"] == 1
+
+    with_error = fv.run_checks(frame, [soft_expectation, hard_expectation])
+    assert with_error["status"] == "fail"
+    assert with_error["passed"] is False
+    assert with_error["summary"]["warnings"] == 1
+    assert with_error["summary"]["errors"] == 1
+    assert {finding["code"] for finding in with_error["findings"]} == {
+        "custom.soft_expectation",
+        "custom.hard_expectation",
+    }
+
+
+def test_custom_check_exceptions_become_structured_failures():
+    frame = pd.DataFrame({"value": [1, 2, 3]})
+
+    @fv.check("exploding rule")
+    def exploding_rule(df):
+        raise RuntimeError("boom")
+
+    result = fv.run_checks(frame, [exploding_rule])
+
+    assert result["status"] == "fail"
+    check_result = result["results"][0]
+    assert check_result["passed"] is False
+    assert check_result["severity"] == "error"
+    assert "RuntimeError: boom" in check_result["execution_error"]
+    assert "RuntimeError" in check_result["message"]
+
+
+def test_each_custom_check_receives_an_isolated_dataframe_copy():
+    frame = pd.DataFrame({"value": [1, 2, 3]})
+
+    @fv.check("mutating check")
+    def mutating_check(df):
+        df["temporary"] = 1
+        return True
+
+    @fv.check("isolation check")
+    def isolation_check(df):
+        return "temporary" not in df.columns
+
+    result = fv.run_checks(frame, [mutating_check, isolation_check])
+
+    assert result["status"] == "pass"
+    assert "temporary" not in frame.columns
+
+
+def test_file_backed_custom_checks_disclose_full_materialization(tmp_path):
+    path = tmp_path / "dataset.csv"
+    pd.DataFrame({"value": [1, 2, 3]}).to_csv(path, index=False)
+
+    result = fv.run_checks(path, [lambda df: bool((df["value"] > 0).all())])
+
+    assert result["status"] == "pass"
+    assert result["execution"]["full_materialization"] is True
+    assert result["execution"]["source"]["kind"] == "file"
+
+
+def test_gate_can_run_only_custom_warning_checks():
+    frame = pd.DataFrame({"latency_ms": [10, 15, 20]})
+
+    @fv.check("latency budget", severity="warning")
+    def latency_budget(df):
+        return {
+            "passed": bool(df["latency_ms"].max() < 15),
+            "message": "Latency exceeded the preferred budget.",
+        }
+
+    result = fv.gate(frame, custom_checks=[latency_budget])
+
+    assert result["status"] == "warn"
+    assert result["passed"] is True
+    assert result["checks_run"] == ["custom"]
+    assert result["checks"]["custom"]["status"] == "warn"
+    assert result["execution"]["custom"]["method"] == "exact_custom_checks"
+    assert result["execution"]["drift"] is None
+    assert result["execution"]["validation"] is None
+    assert "Latency exceeded the preferred budget." in result["reasons"]
+
+
+def test_gate_custom_error_fails_quality_gate():
+    frame = pd.DataFrame({"revenue": [100, -5, 80]})
+
+    @fv.check("positive revenue")
+    def positive_revenue(df):
+        return {
+            "passed": bool((df["revenue"] >= 0).all()),
+            "message": "Negative revenue records are not allowed.",
+        }
+
+    result = fv.gate(frame, custom_checks=[positive_revenue])
+
+    assert result["status"] == "fail"
+    assert result["passed"] is False
+    assert result["checks"]["custom"]["summary"]["errors"] == 1
+    assert "Negative revenue records are not allowed." in result["reasons"]
+
+
+def test_gate_still_requires_at_least_one_check_family():
+    frame = pd.DataFrame({"value": [1]})
+
+    with pytest.raises(ValueError, match="custom_checks"):
+        fv.gate(frame)

From 2240650641379cd4a88595ea853f60a139fe0e9b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:52:10 +0530
Subject: [PATCH 261/439] fix: accept NumPy boolean custom check outcomes

---
 src/framevitals/checks.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/framevitals/checks.py b/src/framevitals/checks.py
index d3c2bb8..747a085 100644
--- a/src/framevitals/checks.py
+++ b/src/framevitals/checks.py
@@ -6,15 +6,16 @@
 from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Literal
+from typing import Any, Literal, cast
 
+import numpy as np
 import pandas as pd
 
 from framevitals.sources import resolve_source
 
 
 CheckSeverity = Literal["warning", "error"]
-CheckFunction = Callable[[pd.DataFrame], bool | Mapping[str, Any]]
+CheckFunction = Callable[[pd.DataFrame], bool | np.bool_ | Mapping[str, Any]]
 DataInput = str | Path | pd.DataFrame
 
 
@@ -26,7 +27,7 @@ def _slug(value: str) -> str:
 def _validate_severity(value: str) -> CheckSeverity:
     if value not in {"warning", "error"}:
         raise ValueError("check severity must be 'warning' or 'error'.")
-    return value  # type: ignore[return-value]
+    return cast(CheckSeverity, value)
 
 
 @dataclass(frozen=True, slots=True)
@@ -45,7 +46,7 @@ def __post_init__(self) -> None:
             raise TypeError("check function must be callable.")
         _validate_severity(self.severity)
 
-    def __call__(self, dataframe: pd.DataFrame) -> bool | Mapping[str, Any]:
+    def __call__(self, dataframe: pd.DataFrame) -> bool | np.bool_ | Mapping[str, Any]:
         return self.function(dataframe)
 
 
@@ -87,10 +88,10 @@ def _normalize_check(value: DataCheck | CheckFunction) -> DataCheck:
 
 def _normalize_outcome(
     definition: DataCheck,
-    raw: bool | Mapping[str, Any],
+    raw: bool | np.bool_ | Mapping[str, Any],
 ) -> dict[str, Any]:
-    if isinstance(raw, bool):
-        passed = raw
+    if isinstance(raw, (bool, np.bool_)):
+        passed = bool(raw)
         message = (
             f"{definition.name} passed."
             if passed

From 745fcde0d6f6181ccc4485b4750cdd887f960ee3 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:52:53 +0530
Subject: [PATCH 262/439] chore: ignore local FrameVitals monitoring state

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index e3f92b4..9391314 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,6 +30,7 @@ frontend/vite.config.js
 frontend/vite.config.d.ts
 
 # Runtime artifacts
+.framevitals/
 uploads/*
 !uploads/.gitkeep
 cleaned/*

From 89b9d6c39b53818149d01d669427ef1f85385371 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:53:25 +0530
Subject: [PATCH 263/439] feat: surface custom checks in gate summaries

---
 src/framevitals/quality_results.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/quality_results.py b/src/framevitals/quality_results.py
index af9d967..d3edf94 100644
--- a/src/framevitals/quality_results.py
+++ b/src/framevitals/quality_results.py
@@ -132,7 +132,7 @@ def summary_text(self) -> str:
 
 
 class GateResult(_QualityResult):
-    """Combined contract/drift quality-gate result."""
+    """Combined contract, custom-check, and drift quality-gate result."""
 
     @property
     def status(self) -> str:
@@ -147,12 +147,18 @@ def reasons(self) -> list[str]:
         value = self.get("reasons", [])
         return value if isinstance(value, list) else []
 
+    @property
+    def checks_run(self) -> list[str]:
+        value = self.get("checks_run", [])
+        return value if isinstance(value, list) else []
+
     def summary_text(self) -> str:
         checks = self.get("checks", {})
         if not isinstance(checks, dict):
             checks = {}
         validation = checks.get("validation")
         drift = checks.get("drift")
+        custom = checks.get("custom")
 
         lines = [
             "FrameVitals quality gate",
@@ -170,6 +176,15 @@ def summary_text(self) -> str:
                     "Drift           "
                     f"{str(drift_gate.get('severity', 'unknown')).upper()}"
                 )
+        if isinstance(custom, dict):
+            summary = custom.get("summary", {})
+            if not isinstance(summary, dict):
+                summary = {}
+            lines.append(
+                "Custom checks   "
+                f"{str(custom.get('status', 'unknown')).upper()} "
+                f"({summary.get('checks', 0)} run)"
+            )
         if self.reasons:
             lines.extend(["", "Reasons"])
             for reason in self.reasons[:10]:

From de2e0263e6496fc9a94c38477b2e71024d04b230 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:53:53 +0530
Subject: [PATCH 264/439] feat: add structured custom check results

---
 src/framevitals/quality_results.py | 45 +++++++++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/src/framevitals/quality_results.py b/src/framevitals/quality_results.py
index d3edf94..10a2303 100644
--- a/src/framevitals/quality_results.py
+++ b/src/framevitals/quality_results.py
@@ -1,4 +1,4 @@
-"""Dict-compatible public results for validation, drift, and quality gates."""
+"""Dict-compatible public results for checks, validation, drift, and quality gates."""
 
 from __future__ import annotations
 
@@ -35,6 +35,49 @@ def to_json(
         return path
 
 
+class CheckResult(_QualityResult):
+    """Dict-compatible result returned by :func:`framevitals.run_checks`."""
+
+    @property
+    def status(self) -> str:
+        return str(self.get("status", "unknown"))
+
+    @property
+    def passed(self) -> bool:
+        return bool(self.get("passed", False))
+
+    @property
+    def findings(self) -> list[dict[str, Any]]:
+        value = self.get("findings", [])
+        return value if isinstance(value, list) else []
+
+    @property
+    def results(self) -> list[dict[str, Any]]:
+        value = self.get("results", [])
+        return value if isinstance(value, list) else []
+
+    def summary_text(self) -> str:
+        summary = self.get("summary", {})
+        if not isinstance(summary, dict):
+            summary = {}
+        lines = [
+            "FrameVitals custom checks",
+            f"Status          {self.status.upper()}",
+            f"Checks          {summary.get('checks', 0)}",
+            f"Passed          {summary.get('passed', 0)}",
+            f"Errors          {summary.get('errors', 0)}",
+            f"Warnings        {summary.get('warnings', 0)}",
+        ]
+        if self.findings:
+            lines.extend(["", "Findings"])
+            for finding in self.findings[:10]:
+                lines.append(
+                    f"- [{str(finding.get('severity', 'error')).upper()}] "
+                    f"{finding.get('title')}: {finding.get('message')}"
+                )
+        return "\n".join(lines)
+
+
 class ValidationResult(_QualityResult):
     """Backward-compatible mapping returned by :func:`framevitals.validate`."""
 

From d4726c8d38a3a3a18e30b13514e8ac9a8a9c0f2f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:54:49 +0530
Subject: [PATCH 265/439] docs: add custom quality gate example

---
 examples/custom_quality_gate.py | 60 +++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 examples/custom_quality_gate.py

diff --git a/examples/custom_quality_gate.py b/examples/custom_quality_gate.py
new file mode 100644
index 0000000..e5ac1c2
--- /dev/null
+++ b/examples/custom_quality_gate.py
@@ -0,0 +1,60 @@
+"""Example: combine FrameVitals contracts, drift, and domain-specific checks."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+import framevitals as fv
+
+
+@fv.check(
+    "positive revenue",
+    severity="error",
+    description="Accounting exports must never contain negative recognized revenue.",
+)
+def positive_revenue(df: pd.DataFrame):
+    invalid = int((df["revenue"] < 0).sum())
+    return {
+        "passed": invalid == 0,
+        "message": (
+            "Revenue values are non-negative."
+            if invalid == 0
+            else f"Found {invalid} negative revenue record(s)."
+        ),
+        "details": {"negative_rows": invalid},
+    }
+
+
+@fv.check("preferred latency budget", severity="warning")
+def latency_budget(df: pd.DataFrame):
+    p95 = float(df["latency_ms"].quantile(0.95))
+    return {
+        "passed": p95 <= 250.0,
+        "message": f"p95 latency is {p95:.1f} ms; preferred maximum is 250 ms.",
+        "details": {"p95_latency_ms": p95},
+    }
+
+
+def main() -> None:
+    reference = pd.read_parquet("data/training.parquet")
+    current = pd.read_parquet("data/production.parquet")
+
+    contract = fv.infer_contract(reference)
+    result = fv.gate(
+        current,
+        reference=reference,
+        contract=contract,
+        custom_checks=[positive_revenue, latency_budget],
+        drift_warn_on="moderate",
+        drift_fail_on="severe",
+    )
+
+    print(result.summary_text())
+    result.to_json("framevitals-gate.json")
+
+    if not result.passed:
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()

From 4af87049486cc1d6efbfb93104b019f654586dce Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:54:56 +0530
Subject: [PATCH 266/439] docs: add snapshot monitoring history example

---
 examples/snapshot_history.py | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 examples/snapshot_history.py

diff --git a/examples/snapshot_history.py b/examples/snapshot_history.py
new file mode 100644
index 0000000..564fefb
--- /dev/null
+++ b/examples/snapshot_history.py
@@ -0,0 +1,34 @@
+"""Example: persist compact FrameVitals monitoring state across recurring runs."""
+
+from __future__ import annotations
+
+import framevitals as fv
+
+
+def main() -> None:
+    report = fv.analyze("data/production.parquet", mode="quick", artifacts=False)
+
+    history = fv.SnapshotHistory(".framevitals/history")
+    path = history.add(report, label="nightly")
+    print(f"Stored snapshot: {path}")
+
+    latest = history.latest()
+    if latest is not None:
+        print(f"Latest fingerprint: {latest['fingerprint']}")
+
+    if len(history) >= 2:
+        change = history.compare_latest()
+        print(f"Health delta: {change['health_delta']}")
+        print(f"New findings: {change['findings']['new']}")
+
+    for point in history.timeline()[-5:]:
+        print(
+            point["created_at"],
+            point["health_score"],
+            point["ml_readiness_score"],
+            point["finding_count"],
+        )
+
+
+if __name__ == "__main__":
+    main()

From 779698e17d1e6aa2904f3a3e2fc1adfe615c5e88 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:57:54 +0530
Subject: [PATCH 267/439] feat: discover third-party check plugins

---
 src/framevitals/plugins.py | 86 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 src/framevitals/plugins.py

diff --git a/src/framevitals/plugins.py b/src/framevitals/plugins.py
new file mode 100644
index 0000000..b9ce2a2
--- /dev/null
+++ b/src/framevitals/plugins.py
@@ -0,0 +1,86 @@
+"""Opt-in discovery for third-party FrameVitals extensions.
+
+FrameVitals never imports installed plugins automatically. Applications that
+want extension discovery must call :func:`discover_checks` explicitly, then
+pass the returned definitions to ``framevitals.run_checks`` or
+``framevitals.gate``.
+
+Third-party packages register one check per Python entry point under the
+``framevitals.checks`` group::
+
+    [project.entry-points."framevitals.checks"]
+    positive_revenue = "acme_data_checks:positive_revenue"
+
+The referenced object must be either a :class:`framevitals.checks.DataCheck`
+or a callable accepting one pandas DataFrame.
+"""
+
+from __future__ import annotations
+
+from importlib import metadata as importlib_metadata
+from typing import Any
+
+from framevitals.checks import DataCheck
+
+
+CHECK_ENTRYPOINT_GROUP = "framevitals.checks"
+
+
+def _entry_points_for(group: str):
+    discovered = importlib_metadata.entry_points()
+    if hasattr(discovered, "select"):
+        return list(discovered.select(group=group))
+    # Compatibility with older importlib.metadata collection objects.
+    return list(discovered.get(group, ()))
+
+
+def _load_entry_point(entry: Any) -> DataCheck:
+    try:
+        loaded = entry.load()
+    except Exception as exc:
+        raise RuntimeError(
+            f"Failed to load FrameVitals check plugin {entry.name!r} from {entry.value!r}."
+        ) from exc
+
+    if isinstance(loaded, DataCheck):
+        return loaded
+    if callable(loaded):
+        return DataCheck(name=str(entry.name), function=loaded)
+    raise TypeError(
+        "FrameVitals check plugin "
+        f"{entry.name!r} must expose a DataCheck or DataFrame callable; "
+        f"got {type(loaded).__name__}."
+    )
+
+
+def discover_checks(
+    *,
+    group: str = CHECK_ENTRYPOINT_GROUP,
+) -> list[DataCheck]:
+    """Load explicitly installed check plugins from a Python entry-point group.
+
+    Discovery is opt-in because loading an entry point executes code from the
+    installed provider package. Returned checks are sorted by registration
+    name for deterministic behavior. Duplicate public check names are rejected
+    so gate behavior cannot depend on package discovery order.
+    """
+    entries = sorted(
+        _entry_points_for(group),
+        key=lambda entry: (str(entry.name), str(entry.value)),
+    )
+    checks: list[DataCheck] = []
+    seen_names: set[str] = set()
+
+    for entry in entries:
+        definition = _load_entry_point(entry)
+        if definition.name in seen_names:
+            raise ValueError(
+                f"Duplicate FrameVitals check plugin name: {definition.name!r}."
+            )
+        seen_names.add(definition.name)
+        checks.append(definition)
+
+    return checks
+
+
+__all__ = ["CHECK_ENTRYPOINT_GROUP", "discover_checks"]

From ac484428e2780f9f333aed5c3b347dbcb172c5ef Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 13:58:06 +0530
Subject: [PATCH 268/439] test: cover third-party check plugin discovery

---
 tests/test_check_plugins.py | 96 +++++++++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 tests/test_check_plugins.py

diff --git a/tests/test_check_plugins.py b/tests/test_check_plugins.py
new file mode 100644
index 0000000..1f846fc
--- /dev/null
+++ b/tests/test_check_plugins.py
@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import pandas as pd
+import pytest
+
+from framevitals.checks import DataCheck
+from framevitals.plugins import CHECK_ENTRYPOINT_GROUP, discover_checks
+
+
+class _FakeEntryPoint:
+    def __init__(self, name, value, loaded=None, error=None):
+        self.name = name
+        self.value = value
+        self._loaded = loaded
+        self._error = error
+
+    def load(self):
+        if self._error is not None:
+            raise self._error
+        return self._loaded
+
+
+class _FakeEntryPoints(list):
+    def select(self, *, group):
+        assert group == CHECK_ENTRYPOINT_GROUP
+        return self
+
+
+def test_discover_checks_loads_callables_and_datachecks(monkeypatch):
+    def positive_values(df: pd.DataFrame):
+        return bool((df["value"] > 0).all())
+
+    explicit = DataCheck(
+        name="explicit check",
+        function=lambda df: True,
+        severity="warning",
+    )
+    entries = _FakeEntryPoints([
+        _FakeEntryPoint("z_plugin", "pkg.z:check", loaded=explicit),
+        _FakeEntryPoint("a_plugin", "pkg.a:positive_values", loaded=positive_values),
+    ])
+    monkeypatch.setattr("framevitals.plugins.importlib_metadata.entry_points", lambda: entries)
+
+    checks = discover_checks()
+
+    assert [item.name for item in checks] == ["a_plugin", "explicit check"]
+    assert checks[0](pd.DataFrame({"value": [1, 2]})) is True
+    assert checks[1].severity == "warning"
+
+
+def test_discover_checks_rejects_duplicate_public_names(monkeypatch):
+    first = DataCheck(name="same", function=lambda df: True)
+    second = DataCheck(name="same", function=lambda df: True)
+    entries = _FakeEntryPoints([
+        _FakeEntryPoint("one", "pkg.one:check", loaded=first),
+        _FakeEntryPoint("two", "pkg.two:check", loaded=second),
+    ])
+    monkeypatch.setattr("framevitals.plugins.importlib_metadata.entry_points", lambda: entries)
+
+    with pytest.raises(ValueError, match="Duplicate FrameVitals check plugin name"):
+        discover_checks()
+
+
+def test_discover_checks_surfaces_plugin_load_failures(monkeypatch):
+    entries = _FakeEntryPoints([
+        _FakeEntryPoint(
+            "broken",
+            "broken_pkg:check",
+            error=ImportError("provider dependency missing"),
+        )
+    ])
+    monkeypatch.setattr("framevitals.plugins.importlib_metadata.entry_points", lambda: entries)
+
+    with pytest.raises(RuntimeError, match="broken"):
+        discover_checks()
+
+
+def test_discover_checks_rejects_unsupported_exports(monkeypatch):
+    entries = _FakeEntryPoints([
+        _FakeEntryPoint("bad", "pkg.bad:value", loaded=42),
+    ])
+    monkeypatch.setattr("framevitals.plugins.importlib_metadata.entry_points", lambda: entries)
+
+    with pytest.raises(TypeError, match="DataCheck or DataFrame callable"):
+        discover_checks()
+
+
+def test_discover_checks_can_target_an_explicit_group(monkeypatch):
+    class _Groups:
+        def select(self, *, group):
+            assert group == "acme.framevitals.checks"
+            return []
+
+    monkeypatch.setattr("framevitals.plugins.importlib_metadata.entry_points", _Groups)
+
+    assert discover_checks(group="acme.framevitals.checks") == []

From b2f61348a7d3d3c18072f5909547197f82d927ed Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:17:20 +0530
Subject: [PATCH 269/439] fix: preserve CSV fallback across Python versions

---
 src/framevitals/sources.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/framevitals/sources.py b/src/framevitals/sources.py
index e387a43..246508c 100644
--- a/src/framevitals/sources.py
+++ b/src/framevitals/sources.py
@@ -162,7 +162,10 @@ def inspect(self) -> DatasetMetadata:
 
         if self._pyarrow_csv() is None:
             self._arrow_compatible = False
-            self._metadata_cache = super().inspect()
+            # Avoid zero-argument super() here: @dataclass(slots=True) may
+            # replace the class object, which breaks the implicit __class__
+            # cell on Python versions where this fallback path is exercised.
+            self._metadata_cache = FileSource.inspect(self)
             return self._metadata_cache
 
         try:
@@ -179,7 +182,7 @@ def inspect(self) -> DatasetMetadata:
             # Preserve existing CSV/TSV compatibility if Arrow cannot parse a
             # file that the pandas loader may still understand.
             self._arrow_compatible = False
-            self._metadata_cache = super().inspect()
+            self._metadata_cache = FileSource.inspect(self)
             return self._metadata_cache
 
         self._arrow_compatible = True
@@ -232,7 +235,7 @@ def iter_batches(
                 yield batch.slice(offset, min(int(batch_size), int(batch.num_rows) - offset))
 
     def load(self) -> pd.DataFrame:
-        return super().load()
+        return FileSource.load(self)
 
 
 @dataclass(slots=True)

From dbf536ff6c8775ddc71c343da042b62576fb3bbd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:18:53 +0530
Subject: [PATCH 270/439] feat: support in-memory Arrow sources

---
 src/framevitals/sources.py | 72 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 70 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/sources.py b/src/framevitals/sources.py
index 246508c..6f3cc69 100644
--- a/src/framevitals/sources.py
+++ b/src/framevitals/sources.py
@@ -82,6 +82,50 @@ def load(self) -> pd.DataFrame:
         return self.dataframe.copy()
 
 
+@dataclass(slots=True)
+class ArrowTableSource:
+    """Projection-aware in-memory Arrow source without eager pandas conversion."""
+
+    table: Any
+    name: str = "<arrow_table>"
+
+    def inspect(self) -> DatasetMetadata:
+        return DatasetMetadata(
+            name=self.name,
+            kind="memory",
+            format="arrow",
+            rows=int(self.table.num_rows),
+            columns=int(self.table.num_columns),
+            size_bytes=int(self.table.nbytes),
+            materialized=True,
+            supports_projection=True,
+            supports_streaming=True,
+        )
+
+    def schema(self):
+        """Return the native Arrow schema without converting row data."""
+        return self.table.schema
+
+    def iter_batches(
+        self,
+        *,
+        batch_size: int = 65_536,
+        columns: Sequence[str] | None = None,
+    ) -> Iterator[Any]:
+        if batch_size < 1:
+            raise ValueError("batch_size must be at least 1.")
+        projected = self.table
+        if columns is not None:
+            projected = projected.select(list(columns))
+        yield from projected.to_batches(max_chunksize=int(batch_size))
+
+    def load(self) -> pd.DataFrame:
+        dataframe = self.table.to_pandas()
+        if dataframe.empty:
+            raise ValueError(f"Dataset is empty: {self.name}")
+        return dataframe
+
+
 @dataclass(slots=True)
 class FileSource:
     path: Path
@@ -305,7 +349,24 @@ def _validate_file_path(path: Path) -> None:
         raise ValueError(f"Expected a dataset file, got: {path}")
 
 
-def resolve_source(data: str | Path | pd.DataFrame | DatasetSource) -> DatasetSource:
+def _arrow_memory_source(data: Any) -> ArrowTableSource | None:
+    """Recognize native Arrow containers without importing Arrow at package import."""
+    try:
+        import pyarrow as pa
+    except ImportError:
+        return None
+
+    if isinstance(data, pa.Table):
+        return ArrowTableSource(data)
+    if isinstance(data, pa.RecordBatch):
+        return ArrowTableSource(
+            pa.Table.from_batches([data]),
+            name="<arrow_record_batch>",
+        )
+    return None
+
+
+def resolve_source(data: Any) -> DatasetSource:
     """Normalize supported user inputs into a DatasetSource implementation."""
     if isinstance(data, pd.DataFrame):
         return PandasSource(data)
@@ -319,6 +380,13 @@ def resolve_source(data: str | Path | pd.DataFrame | DatasetSource) -> DatasetSo
         if suffix == ".tsv":
             return DelimitedTextSource(path, delimiter="\t")
         return FileSource(path)
+
+    arrow_source = _arrow_memory_source(data)
+    if arrow_source is not None:
+        return arrow_source
     if isinstance(data, DatasetSource):
         return data
-    raise TypeError("data must be a pandas DataFrame, dataset path, or DatasetSource.")
+    raise TypeError(
+        "data must be a pandas DataFrame, PyArrow Table/RecordBatch, dataset path, "
+        "or DatasetSource."
+    )

From 47daaa777049e50255e0452c9c7cf3da37cef7f7 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:19:11 +0530
Subject: [PATCH 271/439] refactor: route analysis through dataset sources

---
 src/framevitals/analysis_api.py | 33 ++++++++++++++++++++++-----------
 1 file changed, 22 insertions(+), 11 deletions(-)

diff --git a/src/framevitals/analysis_api.py b/src/framevitals/analysis_api.py
index 0e05057..16f8dae 100644
--- a/src/framevitals/analysis_api.py
+++ b/src/framevitals/analysis_api.py
@@ -1,9 +1,9 @@
 """Public full-analysis dispatcher.
 
 This module keeps input/source routing separate from the legacy analysis API.
-Streaming Arrow-capable sources use the bounded streaming orchestrator when
-artifacts are disabled; DataFrames, non-streaming files, and explicit artifact
-runs retain the existing materialized pipeline behavior.
+Streaming-capable sources use the bounded streaming orchestrator when artifacts
+are disabled; DataFrames and exact/materialized execution retain the existing
+full pipeline behavior.
 """
 
 from __future__ import annotations
@@ -20,7 +20,7 @@
 from framevitals.sources import StreamingDatasetSource, resolve_source
 
 
-DataInput = str | Path | pd.DataFrame
+DataInput = Any
 
 
 def analyze(
@@ -46,6 +46,8 @@ def analyze(
     )
     dataset_id = f"fv_{uuid4().hex[:12]}"
 
+    # Preserve the direct DataFrame path so callers do not pay for a defensive
+    # source-layer copy before the established materialized pipeline begins.
     if isinstance(data, pd.DataFrame):
         if data.empty:
             raise ValueError("Dataset DataFrame is empty.")
@@ -60,7 +62,7 @@ def analyze(
             write_artifacts=resolved.artifacts,
             disabled_modules=resolved.disabled_modules,
         )
-    elif isinstance(data, (str, Path)):
+    else:
         source = resolve_source(data)
         metadata = source.inspect()
         if metadata.rows == 0:
@@ -83,7 +85,7 @@ def analyze(
                 skip_ai=True,
                 disabled_modules=resolved.disabled_modules,
             )
-        else:
+        elif isinstance(data, (str, Path)):
             path = Path(data)
             if not path.exists():
                 raise FileNotFoundError(f"Dataset not found: {path}")
@@ -92,7 +94,7 @@ def analyze(
             payload = run_full_analysis(
                 dataset_id=dataset_id,
                 file_path=path,
-                original_filename=path.name,
+                original_filename=metadata.name,
                 analysis_mode=resolved.mode,
                 target_column=resolved.target,
                 parallel_workers=resolved.workers,
@@ -100,10 +102,19 @@ def analyze(
                 write_artifacts=resolved.artifacts,
                 disabled_modules=resolved.disabled_modules,
             )
-    else:
-        raise TypeError(
-            "data must be a pandas DataFrame or a path to a supported dataset."
-        )
+        else:
+            dataframe = source.load()
+            payload = run_full_analysis(
+                dataset_id=dataset_id,
+                original_filename=metadata.name,
+                analysis_mode=resolved.mode,
+                target_column=resolved.target,
+                parallel_workers=resolved.workers,
+                skip_ai=True,
+                dataframe=dataframe,
+                write_artifacts=resolved.artifacts,
+                disabled_modules=resolved.disabled_modules,
+            )
 
     payload["config"] = resolved.to_dict()
     return AnalysisResult(payload)

From b408aa64a0a151a95f6d3b877e95a51fe07862d1 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:19:27 +0530
Subject: [PATCH 272/439] test: cover in-memory Arrow sources

---
 tests/test_arrow_memory.py | 96 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 tests/test_arrow_memory.py

diff --git a/tests/test_arrow_memory.py b/tests/test_arrow_memory.py
new file mode 100644
index 0000000..8eca58b
--- /dev/null
+++ b/tests/test_arrow_memory.py
@@ -0,0 +1,96 @@
+import numpy as np
+import pytest
+
+pa = pytest.importorskip("pyarrow")
+
+import framevitals
+from framevitals.sources import ArrowTableSource, resolve_source
+
+
+def _table(rows: int = 6_000):
+    values = np.arange(rows, dtype=np.float64)
+    values[::127] = np.nan
+    return pa.table({
+        "value": values,
+        "other": np.arange(rows, dtype=np.float64) * 2.0,
+        "group": [f"g-{index % 5}" for index in range(rows)],
+    })
+
+
+def test_arrow_table_source_exposes_exact_metadata_projection_and_batches():
+    table = _table(2_500)
+    source = resolve_source(table)
+
+    assert isinstance(source, ArrowTableSource)
+    metadata = source.inspect()
+    assert metadata.name == "<arrow_table>"
+    assert metadata.kind == "memory"
+    assert metadata.format == "arrow"
+    assert metadata.rows == table.num_rows
+    assert metadata.columns == table.num_columns
+    assert metadata.size_bytes == table.nbytes
+    assert metadata.materialized is True
+    assert metadata.supports_projection is True
+    assert metadata.supports_streaming is True
+    assert source.schema() == table.schema
+
+    batches = list(source.iter_batches(batch_size=700, columns=["value", "group"]))
+    assert sum(batch.num_rows for batch in batches) == table.num_rows
+    assert max(batch.num_rows for batch in batches) <= 700
+    assert batches[0].schema.names == ["value", "group"]
+
+
+def test_public_profile_streams_arrow_table_without_pandas_materialization(monkeypatch):
+    table = _table(12_000)
+
+    def fail_load(self):
+        raise AssertionError("Arrow profile must not materialize the complete table in pandas")
+
+    monkeypatch.setattr(ArrowTableSource, "load", fail_load)
+    result = framevitals.profile(table)
+
+    assert result["dataset_name"] == "<arrow_table>"
+    assert result["shape"] == {"rows": table.num_rows, "columns": table.num_columns}
+    assert result["streaming_metadata"]["enabled"] is True
+    assert result["streaming_metadata"]["full_materialization"] is False
+    assert result["source_metadata"]["kind"] == "memory"
+    assert result["source_metadata"]["format"] == "arrow"
+    assert result["missing_counts"]["value"] == int(np.isnan(table["value"].to_numpy()).sum())
+
+
+def test_public_analyze_dispatches_arrow_table_through_streaming_pipeline(monkeypatch):
+    table = _table(6_000)
+
+    def fail_load(self):
+        raise AssertionError("Arrow analysis must not materialize the complete table in pandas")
+
+    monkeypatch.setattr(ArrowTableSource, "load", fail_load)
+    result = framevitals.analyze(table, mode="quick", artifacts=False, workers=1)
+
+    assert result["filename"] == "<arrow_table>"
+    assert result["profile"]["shape"] == {
+        "rows": table.num_rows,
+        "columns": table.num_columns,
+    }
+    assert result["execution"]["streaming"]["enabled"] is True
+    assert result["execution"]["streaming"]["full_materialization"] is False
+    assert result["execution"]["streaming"]["source_rows"] == table.num_rows
+
+
+def test_arrow_record_batch_normalizes_to_streaming_source():
+    batch = pa.record_batch({
+        "value": [1.0, 2.0, 3.0],
+        "label": ["a", "b", "a"],
+    })
+    source = resolve_source(batch)
+    metadata = source.inspect()
+
+    assert isinstance(source, ArrowTableSource)
+    assert metadata.name == "<arrow_record_batch>"
+    assert metadata.rows == 3
+    assert metadata.columns == 2
+    assert metadata.supports_streaming is True
+
+    result = framevitals.profile(batch)
+    assert result["dataset_name"] == "<arrow_record_batch>"
+    assert result["shape"] == {"rows": 3, "columns": 2}

From f4c27c69c8325aa99a65055555b7c5184b7db0fd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:19:54 +0530
Subject: [PATCH 273/439] ci: cover in-memory Arrow inputs

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6a7dda2..c20b566 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -237,7 +237,7 @@ jobs:
       - name: Run streaming source tests without native extension
         env:
           FRAMEVITALS_BACKEND: numpy
-        run: pytest -v tests/test_parquet_streaming.py tests/test_statistics_streaming.py tests/test_streaming_drift.py tests/test_csv_streaming.py tests/test_cli_gate_streaming.py
+        run: pytest -v tests/test_parquet_streaming.py tests/test_statistics_streaming.py tests/test_streaming_drift.py tests/test_csv_streaming.py tests/test_arrow_memory.py tests/test_cli_gate_streaming.py
 
   optional-features:
     name: optional features (Python 3.12)

From fec8290f16880ae4c3c9f5f7d30fbd7bcfd79b45 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:22:01 +0530
Subject: [PATCH 274/439] feat: accept Arrow PyCapsule table inputs

---
 src/framevitals/sources.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/sources.py b/src/framevitals/sources.py
index 6f3cc69..096bc6b 100644
--- a/src/framevitals/sources.py
+++ b/src/framevitals/sources.py
@@ -350,7 +350,7 @@ def _validate_file_path(path: Path) -> None:
 
 
 def _arrow_memory_source(data: Any) -> ArrowTableSource | None:
-    """Recognize native Arrow containers without importing Arrow at package import."""
+    """Recognize Arrow containers/producers without importing Arrow at package import."""
     try:
         import pyarrow as pa
     except ImportError:
@@ -363,6 +363,19 @@ def _arrow_memory_source(data: Any) -> ArrowTableSource | None:
             pa.Table.from_batches([data]),
             name="<arrow_record_batch>",
         )
+    if isinstance(data, pa.RecordBatchReader):
+        raise TypeError(
+            "Arrow RecordBatchReader inputs do not expose a cheap exact row count. "
+            "Pass a PyArrow Table/RecordBatch or another materialized Arrow-compatible "
+            "table object instead."
+        )
+
+    arrow_stream = getattr(data, "__arrow_c_stream__", None)
+    if callable(arrow_stream):
+        return ArrowTableSource(
+            pa.table(data),
+            name="<arrow_capsule_table>",
+        )
     return None
 
 
@@ -387,6 +400,6 @@ def resolve_source(data: Any) -> DatasetSource:
     if isinstance(data, DatasetSource):
         return data
     raise TypeError(
-        "data must be a pandas DataFrame, PyArrow Table/RecordBatch, dataset path, "
+        "data must be a pandas DataFrame, Arrow-compatible table, dataset path, "
         "or DatasetSource."
     )

From 9d3934c2f80137507ddf146af02a2ccd273a6569 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:22:17 +0530
Subject: [PATCH 275/439] test: cover Arrow capsule interoperability

---
 tests/test_arrow_memory.py | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/tests/test_arrow_memory.py b/tests/test_arrow_memory.py
index 8eca58b..a03396b 100644
--- a/tests/test_arrow_memory.py
+++ b/tests/test_arrow_memory.py
@@ -94,3 +94,32 @@ def test_arrow_record_batch_normalizes_to_streaming_source():
     result = framevitals.profile(batch)
     assert result["dataset_name"] == "<arrow_record_batch>"
     assert result["shape"] == {"rows": 3, "columns": 2}
+
+
+def test_arrow_capsule_table_normalizes_without_library_specific_adapter():
+    table = _table(1_200)
+
+    class CapsuleTable:
+        def __arrow_c_stream__(self, requested_schema=None):
+            return table.__arrow_c_stream__(requested_schema)
+
+    source = resolve_source(CapsuleTable())
+    metadata = source.inspect()
+
+    assert isinstance(source, ArrowTableSource)
+    assert metadata.name == "<arrow_capsule_table>"
+    assert metadata.rows == table.num_rows
+    assert metadata.columns == table.num_columns
+    assert metadata.supports_streaming is True
+
+    result = framevitals.profile(CapsuleTable())
+    assert result["shape"] == {"rows": table.num_rows, "columns": table.num_columns}
+    assert result["source_metadata"]["format"] == "arrow"
+
+
+def test_arrow_record_batch_reader_is_not_silently_materialized():
+    table = _table(20)
+    reader = pa.RecordBatchReader.from_batches(table.schema, table.to_batches())
+
+    with pytest.raises(TypeError, match="cheap exact row count"):
+        resolve_source(reader)

From 07f1f86044b81ed36bbfd7277496cb25922b6a7d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:24:04 +0530
Subject: [PATCH 276/439] feat: add streaming DuckDB relation source

---
 src/framevitals/duckdb_source.py | 118 +++++++++++++++++++++++++++++++
 1 file changed, 118 insertions(+)
 create mode 100644 src/framevitals/duckdb_source.py

diff --git a/src/framevitals/duckdb_source.py b/src/framevitals/duckdb_source.py
new file mode 100644
index 0000000..7a53343
--- /dev/null
+++ b/src/framevitals/duckdb_source.py
@@ -0,0 +1,118 @@
+"""Optional DuckDB relation adapter for FrameVitals source-aware execution."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+import pandas as pd
+
+from framevitals.sources import DatasetMetadata
+
+
+def _quoted_identifier(name: str) -> str:
+    """Quote a DuckDB identifier without treating column names as expressions."""
+    return '"' + str(name).replace('"', '""') + '"'
+
+
+@dataclass(slots=True)
+class DuckDBRelationSource:
+    """Stream a lazy DuckDB relation through Arrow record batches.
+
+    ``inspect()`` executes an exact ``count(*)`` once and caches the result.
+    This can scan the underlying relation, but it does not materialize all rows
+    in Python memory. Row data is consumed later through ``to_arrow_reader``.
+    """
+
+    relation: Any
+    name: str = "<duckdb_relation>"
+    _metadata_cache: DatasetMetadata | None = field(default=None, init=False, repr=False)
+    _schema_cache: Any = field(default=None, init=False, repr=False)
+
+    def inspect(self) -> DatasetMetadata:
+        if self._metadata_cache is not None:
+            return self._metadata_cache
+
+        columns = list(self.relation.columns)
+        count_row = self.relation.aggregate(
+            "count(*) AS __framevitals_rows"
+        ).fetchone()
+        if not count_row:
+            raise ValueError("DuckDB relation did not return a row count.")
+
+        rows = int(count_row[0])
+        schema = self.schema()
+        self._metadata_cache = DatasetMetadata(
+            name=self.name,
+            kind="relation",
+            format="duckdb",
+            rows=rows,
+            columns=len(columns),
+            size_bytes=None,
+            materialized=False,
+            supports_projection=True,
+            supports_streaming=True,
+        )
+        if len(schema) != len(columns):
+            raise ValueError(
+                "DuckDB relation schema changed while FrameVitals inspected it."
+            )
+        return self._metadata_cache
+
+    def schema(self):
+        """Return an Arrow schema without fetching relation rows."""
+        if self._schema_cache is None:
+            self._schema_cache = self.relation.limit(0).to_arrow_table().schema
+        return self._schema_cache
+
+    def iter_batches(
+        self,
+        *,
+        batch_size: int = 65_536,
+        columns: Sequence[str] | None = None,
+    ) -> Iterator[Any]:
+        if batch_size < 1:
+            raise ValueError("batch_size must be at least 1.")
+
+        projected = self.relation
+        if columns is not None:
+            available = set(self.relation.columns)
+            missing = [column for column in columns if column not in available]
+            if missing:
+                raise KeyError(
+                    "DuckDB relation does not contain requested column(s): "
+                    + ", ".join(map(str, missing))
+                )
+            projected = self.relation.select(
+                *[_quoted_identifier(column) for column in columns]
+            )
+
+        reader = projected.to_arrow_reader(int(batch_size))
+        yield from reader
+
+    def load(self) -> pd.DataFrame:
+        """Materialize the complete relation only for exact/full-row APIs."""
+        dataframe = self.relation.df()
+        if dataframe.empty:
+            raise ValueError("Dataset is empty: <duckdb_relation>")
+        return dataframe
+
+
+def resolve_duckdb_source(data: Any) -> DuckDBRelationSource | None:
+    """Recognize DuckDB relations without importing DuckDB for normal inputs."""
+    data_type = type(data)
+    if data_type.__name__ != "DuckDBPyRelation":
+        return None
+    if not data_type.__module__.startswith(("duckdb", "_duckdb")):
+        return None
+
+    try:
+        import duckdb
+    except ImportError:
+        return None
+
+    relation_type = getattr(duckdb, "DuckDBPyRelation", None)
+    if relation_type is None or not isinstance(data, relation_type):
+        return None
+    return DuckDBRelationSource(data)

From 4859075530e633e60e802bffea8955efbcef2c99 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:24:35 +0530
Subject: [PATCH 277/439] feat: route DuckDB relations through source adapter

---
 src/framevitals/sources.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/sources.py b/src/framevitals/sources.py
index 096bc6b..b4e1f39 100644
--- a/src/framevitals/sources.py
+++ b/src/framevitals/sources.py
@@ -379,6 +379,19 @@ def _arrow_memory_source(data: Any) -> ArrowTableSource | None:
     return None
 
 
+def _duckdb_relation_source(data: Any):
+    """Recognize lazy DuckDB relations before generic Arrow conversion."""
+    data_type = type(data)
+    if data_type.__name__ != "DuckDBPyRelation":
+        return None
+    if not data_type.__module__.startswith(("duckdb", "_duckdb")):
+        return None
+
+    from framevitals.duckdb_source import resolve_duckdb_source
+
+    return resolve_duckdb_source(data)
+
+
 def resolve_source(data: Any) -> DatasetSource:
     """Normalize supported user inputs into a DatasetSource implementation."""
     if isinstance(data, pd.DataFrame):
@@ -394,12 +407,16 @@ def resolve_source(data: Any) -> DatasetSource:
             return DelimitedTextSource(path, delimiter="\t")
         return FileSource(path)
 
+    duckdb_source = _duckdb_relation_source(data)
+    if duckdb_source is not None:
+        return duckdb_source
+
     arrow_source = _arrow_memory_source(data)
     if arrow_source is not None:
         return arrow_source
     if isinstance(data, DatasetSource):
         return data
     raise TypeError(
-        "data must be a pandas DataFrame, Arrow-compatible table, dataset path, "
-        "or DatasetSource."
+        "data must be a pandas DataFrame, Arrow-compatible table, DuckDB relation, "
+        "dataset path, or DatasetSource."
     )

From f3215e4c47ef6e63a9959ef3df1e990227645e21 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:24:52 +0530
Subject: [PATCH 278/439] feat: add optional DuckDB interoperability extra

---
 pyproject.toml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index 5ceb18e..8479677 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -67,6 +67,10 @@ web = [
 arrow = [
     "pyarrow>=25.0,<26",
 ]
+duckdb = [
+    "duckdb>=1.4",
+    "pyarrow>=25.0,<26",
+]
 all = [
     "openpyxl>=3.1",
     "xlrd>=2.0",
@@ -82,6 +86,7 @@ all = [
     "gunicorn>=22.0",
     "werkzeug>=3.0",
     "pyarrow>=25.0,<26",
+    "duckdb>=1.4",
 ]
 dev = [
     "pytest>=8.2",

From 9686ee08300ee2d91fa27f6331da23c34c082166 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:25:11 +0530
Subject: [PATCH 279/439] test: cover streaming DuckDB relation inputs

---
 tests/test_duckdb_source.py | 112 ++++++++++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)
 create mode 100644 tests/test_duckdb_source.py

diff --git a/tests/test_duckdb_source.py b/tests/test_duckdb_source.py
new file mode 100644
index 0000000..707cc71
--- /dev/null
+++ b/tests/test_duckdb_source.py
@@ -0,0 +1,112 @@
+import pytest
+
+pytest.importorskip("pyarrow")
+duckdb = pytest.importorskip("duckdb")
+
+import framevitals
+from framevitals.duckdb_source import DuckDBRelationSource
+from framevitals.sources import resolve_source
+
+
+def _relation(rows: int = 12_000):
+    connection = duckdb.connect()
+    relation = connection.sql(
+        f"""
+        SELECT
+            range::DOUBLE AS value,
+            (range * 2)::DOUBLE AS other,
+            concat('g-', range % 5) AS grp
+        FROM range({int(rows)})
+        """
+    )
+    return connection, relation
+
+
+def test_duckdb_relation_source_exposes_exact_metadata_projection_and_batches():
+    connection, relation = _relation(4_000)
+    try:
+        source = resolve_source(relation)
+        assert isinstance(source, DuckDBRelationSource)
+
+        metadata = source.inspect()
+        assert metadata.name == "<duckdb_relation>"
+        assert metadata.kind == "relation"
+        assert metadata.format == "duckdb"
+        assert metadata.rows == 4_000
+        assert metadata.columns == 3
+        assert metadata.size_bytes is None
+        assert metadata.materialized is False
+        assert metadata.supports_projection is True
+        assert metadata.supports_streaming is True
+        assert source.schema().names == ["value", "other", "grp"]
+
+        batches = list(source.iter_batches(batch_size=700, columns=["value", "grp"]))
+        assert sum(batch.num_rows for batch in batches) == 4_000
+        assert max(batch.num_rows for batch in batches) <= 700
+        assert batches[0].schema.names == ["value", "grp"]
+    finally:
+        connection.close()
+
+
+def test_public_profile_streams_duckdb_relation_without_pandas_materialization(monkeypatch):
+    connection, relation = _relation(12_000)
+
+    def fail_load(self):
+        raise AssertionError("DuckDB profile must not materialize the complete relation")
+
+    monkeypatch.setattr(DuckDBRelationSource, "load", fail_load)
+    try:
+        result = framevitals.profile(relation)
+    finally:
+        connection.close()
+
+    assert result["dataset_name"] == "<duckdb_relation>"
+    assert result["shape"] == {"rows": 12_000, "columns": 3}
+    assert result["streaming_metadata"]["enabled"] is True
+    assert result["streaming_metadata"]["full_materialization"] is False
+    assert result["source_metadata"]["kind"] == "relation"
+    assert result["source_metadata"]["format"] == "duckdb"
+
+
+def test_public_analyze_streams_duckdb_relation(monkeypatch):
+    connection, relation = _relation(6_000)
+
+    def fail_load(self):
+        raise AssertionError("DuckDB analysis must stay on the streaming relation path")
+
+    monkeypatch.setattr(DuckDBRelationSource, "load", fail_load)
+    try:
+        result = framevitals.analyze(
+            relation,
+            mode="quick",
+            artifacts=False,
+            workers=1,
+        )
+    finally:
+        connection.close()
+
+    assert result["filename"] == "<duckdb_relation>"
+    assert result["profile"]["shape"] == {"rows": 6_000, "columns": 3}
+    assert result["execution"]["streaming"]["enabled"] is True
+    assert result["execution"]["streaming"]["full_materialization"] is False
+    assert result["execution"]["streaming"]["source_rows"] == 6_000
+
+
+def test_plan_uses_bounded_duckdb_relation_sample(monkeypatch):
+    connection, relation = _relation(20_000)
+
+    def fail_load(self):
+        raise AssertionError("DuckDB planning must not load the complete relation")
+
+    monkeypatch.setattr(DuckDBRelationSource, "load", fail_load)
+    try:
+        plan = framevitals.plan(relation, mode="standard")
+    finally:
+        connection.close()
+
+    assert plan["dataset_name"] == "<duckdb_relation>"
+    assert plan["source"]["format"] == "duckdb"
+    assert plan["shape"] == {"rows": 20_000, "columns": 3}
+    assert plan["planning_data"]["materialized_full_dataset"] is False
+    assert plan["planning_data"]["sampled"] is True
+    assert plan["planning_data"]["sample_rows"] == 5_000

From 50b93464e0c761a82da7481781f45f6eb8b6cb71 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:25:28 +0530
Subject: [PATCH 280/439] ci: add Arrow and DuckDB interoperability checks

---
 .github/workflows/interop.yml | 62 +++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 .github/workflows/interop.yml

diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml
new file mode 100644
index 0000000..d3a2462
--- /dev/null
+++ b/.github/workflows/interop.yml
@@ -0,0 +1,62 @@
+name: Interoperability
+
+on:
+  push:
+    branches: [main, dev, "develop/august"]
+    paths:
+      - "src/framevitals/sources.py"
+      - "src/framevitals/duckdb_source.py"
+      - "src/framevitals/analysis_api.py"
+      - "src/framevitals/streaming_*.py"
+      - "tests/test_arrow_memory.py"
+      - "tests/test_duckdb_source.py"
+      - "pyproject.toml"
+      - ".github/workflows/interop.yml"
+  pull_request:
+    paths:
+      - "src/framevitals/sources.py"
+      - "src/framevitals/duckdb_source.py"
+      - "src/framevitals/analysis_api.py"
+      - "src/framevitals/streaming_*.py"
+      - "tests/test_arrow_memory.py"
+      - "tests/test_duckdb_source.py"
+      - "pyproject.toml"
+      - ".github/workflows/interop.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  arrow-duckdb:
+    name: Arrow + DuckDB (Python 3.12)
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install interoperability capabilities
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[duckdb,dev]"
+
+      - name: Verify dependency consistency
+        run: python -m pip check
+
+      - name: Run interoperability tests
+        env:
+          FRAMEVITALS_BACKEND: numpy
+        run: pytest -v tests/test_arrow_memory.py tests/test_duckdb_source.py

From ec9b261f82f578fcc56f4069f6f14c5874f0608b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:26:34 +0530
Subject: [PATCH 281/439] fix: report source materialization by execution
 semantics

---
 src/framevitals/operations.py | 52 +++++++++++++++++++++++------------
 1 file changed, 35 insertions(+), 17 deletions(-)

diff --git a/src/framevitals/operations.py b/src/framevitals/operations.py
index a24da05..a3d74cb 100644
--- a/src/framevitals/operations.py
+++ b/src/framevitals/operations.py
@@ -26,7 +26,7 @@
 from framevitals.sources import DatasetMetadata, StreamingDatasetSource, resolve_source
 
 
-DataInput = str | Path | pd.DataFrame
+DataInput = Any
 _DRIFT_SEVERITY_RANK = {"stable": 0, "minor": 1, "moderate": 2, "severe": 3}
 _DRIFT_SAMPLE_ROWS = 50_000
 
@@ -58,6 +58,11 @@ def _load_input(value: DataInput, *, label: str) -> tuple[pd.DataFrame, str]:
     return dataframe, metadata.name
 
 
+def _load_fully_materializes_source(metadata: DatasetMetadata) -> bool:
+    """Whether ``source.load()`` creates a complete pandas representation."""
+    return not (metadata.kind == "memory" and metadata.format == "pandas")
+
+
 def _comparison_frame(
     source,
     metadata: DatasetMetadata,
@@ -96,7 +101,7 @@ def _comparison_frame(
         "sample_rows": int(len(dataframe)),
         "sampled": False,
         "strategy": "full_input",
-        "full_materialization": bool(metadata.kind == "file"),
+        "full_materialization": _load_fully_materializes_source(metadata),
         "source": metadata.to_dict(),
     }
 
@@ -231,7 +236,7 @@ def validate(
     payload["dataset_name"] = metadata.name
     payload["execution"] = {
         "method": "exact_contract_validation",
-        "full_materialization": bool(metadata.kind == "file"),
+        "full_materialization": _load_fully_materializes_source(metadata),
         "source": metadata.to_dict(),
         "reason": (
             "Contract validation remains exact; uniqueness, allowed-value, and bound "
@@ -362,6 +367,27 @@ def fail() -> None:
             if message and message not in reasons:
                 reasons.append(message)
 
+    validation_execution = (
+        checks.get("validation", {}).get("execution")
+        if isinstance(checks.get("validation"), Mapping)
+        else None
+    )
+    drift_execution = (
+        checks.get("drift", {}).get("execution")
+        if isinstance(checks.get("drift"), Mapping)
+        else None
+    )
+    custom_execution = (
+        checks.get("custom", {}).get("execution")
+        if isinstance(checks.get("custom"), Mapping)
+        else None
+    )
+    execution_blocks = [
+        block
+        for block in (validation_execution, drift_execution, custom_execution)
+        if isinstance(block, Mapping)
+    ]
+
     return GateResult({
         "status": status,
         "passed": status != "fail",
@@ -375,20 +401,12 @@ def fail() -> None:
         "reasons": reasons,
         "checks": checks,
         "execution": {
-            "validation": (
-                checks.get("validation", {}).get("execution")
-                if isinstance(checks.get("validation"), Mapping)
-                else None
-            ),
-            "drift": (
-                checks.get("drift", {}).get("execution")
-                if isinstance(checks.get("drift"), Mapping)
-                else None
-            ),
-            "custom": (
-                checks.get("custom", {}).get("execution")
-                if isinstance(checks.get("custom"), Mapping)
-                else None
+            "full_materialization": any(
+                bool(block.get("full_materialization"))
+                for block in execution_blocks
             ),
+            "validation": validation_execution,
+            "drift": drift_execution,
+            "custom": custom_execution,
         },
     })

From 02d1f81c71b36b920aba46afd4a354c58c00eae6 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:27:03 +0530
Subject: [PATCH 282/439] fix: align custom check results and materialization
 provenance

---
 src/framevitals/checks.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/framevitals/checks.py b/src/framevitals/checks.py
index 747a085..0c8348f 100644
--- a/src/framevitals/checks.py
+++ b/src/framevitals/checks.py
@@ -5,18 +5,18 @@
 import re
 from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass
-from pathlib import Path
 from typing import Any, Literal, cast
 
 import numpy as np
 import pandas as pd
 
+from framevitals.quality_results import CheckResult
 from framevitals.sources import resolve_source
 
 
 CheckSeverity = Literal["warning", "error"]
 CheckFunction = Callable[[pd.DataFrame], bool | np.bool_ | Mapping[str, Any]]
-DataInput = str | Path | pd.DataFrame
+DataInput = Any
 
 
 def _slug(value: str) -> str:
@@ -30,6 +30,11 @@ def _validate_severity(value: str) -> CheckSeverity:
     return cast(CheckSeverity, value)
 
 
+def _load_fully_materializes_source(metadata) -> bool:
+    """Whether exact custom checks create a complete pandas representation."""
+    return not (metadata.kind == "memory" and metadata.format == "pandas")
+
+
 @dataclass(frozen=True, slots=True)
 class DataCheck:
     """Named user-defined data check with gate severity metadata."""
@@ -130,11 +135,11 @@ def _normalize_outcome(
 def run_checks(
     data: DataInput,
     checks: Sequence[DataCheck | CheckFunction],
-) -> dict[str, Any]:
+) -> CheckResult:
     """Run user-defined checks exactly against a materialized DataFrame.
 
     Custom Python callables can inspect arbitrary row-level relationships, so
-    FrameVitals does not silently sample their input. File-backed inputs are
+    FrameVitals does not silently sample their input. Non-pandas sources are
     materialized intentionally and that decision is disclosed in ``execution``.
     """
     definitions = [_normalize_check(value) for value in checks]
@@ -184,7 +189,7 @@ def run_checks(
         for result in failures
     ]
 
-    return {
+    return CheckResult({
         "status": status,
         "passed": status != "fail",
         "results": results,
@@ -197,11 +202,11 @@ def run_checks(
         },
         "execution": {
             "method": "exact_custom_checks",
-            "full_materialization": bool(metadata.kind == "file"),
+            "full_materialization": _load_fully_materializes_source(metadata),
             "source": metadata.to_dict(),
             "reason": (
                 "Arbitrary custom Python checks run on the complete DataFrame; "
                 "FrameVitals does not silently sample user-defined invariants."
             ),
         },
-    }
+    })

From 4ab16d5a019037680ad27842bb0903051d2488b6 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:28:10 +0530
Subject: [PATCH 283/439] feat: expose checks and plugin discovery at package
 root

---
 src/framevitals/__init__.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 7ca4e08..4e6193a 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -6,7 +6,12 @@
 
 from framevitals.config import AnalysisConfig, available_modules
 from framevitals.planning import AnalysisPlan
-from framevitals.quality_results import DriftResult, GateResult, ValidationResult
+from framevitals.quality_results import (
+    CheckResult,
+    DriftResult,
+    GateResult,
+    ValidationResult,
+)
 from framevitals.result import AnalysisResult, ColumnResult
 from framevitals.snapshots import (
     AnalysisSnapshot,
@@ -163,7 +168,7 @@ def analyze(
     config: Any = None,
     disabled_modules: list[str] | tuple[str, ...] | None = None,
 ) -> AnalysisResult:
-    """Analyze a tabular dataset from a DataFrame or supported file path."""
+    """Analyze a supported tabular source through the canonical dispatcher."""
     from framevitals.analysis_api import analyze as _analyze
 
     return _analyze(
@@ -281,13 +286,20 @@ def check(
     )
 
 
-def run_checks(data: Any, checks: Any) -> dict[str, Any]:
+def run_checks(data: Any, checks: Any) -> CheckResult:
     """Run custom data checks exactly against a dataset."""
     from framevitals.checks import run_checks as _run_checks
 
     return _run_checks(data, checks)
 
 
+def discover_checks(*, group: str = "framevitals.checks"):
+    """Explicitly load installed third-party check plugins."""
+    from framevitals.plugins import discover_checks as _discover_checks
+
+    return _discover_checks(group=group)
+
+
 def gate(
     current: Any,
     *,
@@ -325,6 +337,7 @@ def gate(
     "CleaningPlan",
     "ColumnResult",
     "DataCheck",
+    "CheckResult",
     "DriftResult",
     "GateResult",
     "ValidationResult",
@@ -347,6 +360,7 @@ def gate(
     "validate",
     "check",
     "run_checks",
+    "discover_checks",
     "gate",
     "available_modules",
     "create_snapshot",

From c78a45f5b8d617e35af88f6c572b3a7494b003bd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:30:07 +0530
Subject: [PATCH 284/439] test: lock custom check result contract

---
 tests/test_custom_checks.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/test_custom_checks.py b/tests/test_custom_checks.py
index 9bbdf3e..10d8f41 100644
--- a/tests/test_custom_checks.py
+++ b/tests/test_custom_checks.py
@@ -43,11 +43,14 @@ def positive_revenue(df):
     assert positive_revenue.name == "positive revenue"
     result = fv.run_checks(frame, [positive_revenue])
 
-    assert result["status"] == "pass"
-    assert result["passed"] is True
+    assert isinstance(result, fv.CheckResult)
+    assert result.status == "pass"
+    assert result.passed is True
     assert result["summary"] == {"checks": 1, "passed": 1, "warnings": 0, "errors": 0}
-    assert result["results"][0]["code"] == "custom.positive_revenue"
-    assert result["results"][0]["details"] == {"minimum": 10.0}
+    assert result.results[0]["code"] == "custom.positive_revenue"
+    assert result.results[0]["details"] == {"minimum": 10.0}
+    assert result.findings == []
+    assert "FrameVitals custom checks" in result.summary_text()
     assert result["execution"]["method"] == "exact_custom_checks"
     assert result["execution"]["full_materialization"] is False
 
@@ -141,6 +144,7 @@ def latency_budget(df):
     assert result["passed"] is True
     assert result["checks_run"] == ["custom"]
     assert result["checks"]["custom"]["status"] == "warn"
+    assert result["execution"]["full_materialization"] is False
     assert result["execution"]["custom"]["method"] == "exact_custom_checks"
     assert result["execution"]["drift"] is None
     assert result["execution"]["validation"] is None

From 561658cb83a452b135d6b84c1bc39dce35968c7b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:30:22 +0530
Subject: [PATCH 285/439] test: expose plugin discovery through public API

---
 tests/test_check_plugins.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/test_check_plugins.py b/tests/test_check_plugins.py
index 1f846fc..bb3024d 100644
--- a/tests/test_check_plugins.py
+++ b/tests/test_check_plugins.py
@@ -3,6 +3,7 @@
 import pandas as pd
 import pytest
 
+import framevitals as fv
 from framevitals.checks import DataCheck
 from framevitals.plugins import CHECK_ENTRYPOINT_GROUP, discover_checks
 
@@ -48,6 +49,22 @@ def positive_values(df: pd.DataFrame):
     assert checks[1].severity == "warning"
 
 
+def test_public_discover_checks_delegates_to_opt_in_plugin_loader(monkeypatch):
+    def positive_values(df: pd.DataFrame):
+        return bool((df["value"] > 0).all())
+
+    entries = _FakeEntryPoints([
+        _FakeEntryPoint("positive_values", "pkg:positive_values", loaded=positive_values),
+    ])
+    monkeypatch.setattr("framevitals.plugins.importlib_metadata.entry_points", lambda: entries)
+
+    checks = fv.discover_checks()
+
+    assert len(checks) == 1
+    assert isinstance(checks[0], fv.DataCheck)
+    assert checks[0].name == "positive_values"
+
+
 def test_discover_checks_rejects_duplicate_public_names(monkeypatch):
     first = DataCheck(name="same", function=lambda df: True)
     second = DataCheck(name="same", function=lambda df: True)

From 4eb536dd1f42f9f9c1db09b84c3724e364bc4f47 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:30:45 +0530
Subject: [PATCH 286/439] test: verify DuckDB materialization provenance

---
 tests/test_duckdb_source.py | 60 +++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/tests/test_duckdb_source.py b/tests/test_duckdb_source.py
index 707cc71..5a1a839 100644
--- a/tests/test_duckdb_source.py
+++ b/tests/test_duckdb_source.py
@@ -1,3 +1,4 @@
+import pandas as pd
 import pytest
 
 pytest.importorskip("pyarrow")
@@ -22,6 +23,14 @@ def _relation(rows: int = 12_000):
     return connection, relation
 
 
+def _frame(rows: int):
+    return pd.DataFrame({
+        "value": [float(index) for index in range(rows)],
+        "other": [float(index * 2) for index in range(rows)],
+        "grp": [f"g-{index % 5}" for index in range(rows)],
+    })
+
+
 def test_duckdb_relation_source_exposes_exact_metadata_projection_and_batches():
     connection, relation = _relation(4_000)
     try:
@@ -110,3 +119,54 @@ def fail_load(self):
     assert plan["planning_data"]["materialized_full_dataset"] is False
     assert plan["planning_data"]["sampled"] is True
     assert plan["planning_data"]["sample_rows"] == 5_000
+
+
+def test_exact_contract_validation_reports_duckdb_materialization():
+    rows = 100
+    contract = framevitals.infer_contract(_frame(rows))
+    connection, relation = _relation(rows)
+    try:
+        result = framevitals.validate(relation, contract)
+    finally:
+        connection.close()
+
+    assert result["status"] in {"pass", "warn"}
+    assert result["execution"]["method"] == "exact_contract_validation"
+    assert result["execution"]["full_materialization"] is True
+    assert result["execution"]["source"]["format"] == "duckdb"
+
+
+def test_exact_custom_checks_report_duckdb_materialization():
+    connection, relation = _relation(100)
+    try:
+        result = framevitals.run_checks(
+            relation,
+            [lambda df: bool((df["value"] >= 0).all())],
+        )
+    finally:
+        connection.close()
+
+    assert isinstance(result, framevitals.CheckResult)
+    assert result.status == "pass"
+    assert result["execution"]["full_materialization"] is True
+    assert result["execution"]["source"]["format"] == "duckdb"
+
+
+def test_drift_only_gate_keeps_duckdb_relations_streaming(monkeypatch):
+    reference_connection, reference = _relation(1_000)
+    current_connection, current = _relation(1_000)
+
+    def fail_load(self):
+        raise AssertionError("Drift-only DuckDB gate must stay on the streaming path")
+
+    monkeypatch.setattr(DuckDBRelationSource, "load", fail_load)
+    try:
+        result = framevitals.gate(current, reference=reference)
+    finally:
+        reference_connection.close()
+        current_connection.close()
+
+    assert result.status in {"pass", "warn"}
+    assert result["checks_run"] == ["drift"]
+    assert result["execution"]["full_materialization"] is False
+    assert result["execution"]["drift"]["full_materialization"] is False

From 497b3b8eeb474534e0b1f0667080c950fa3f1b9b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:31:13 +0530
Subject: [PATCH 287/439] feat: mirror check discovery in compatibility API

---
 src/framevitals/api.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index 6014e99..99f7b80 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -17,7 +17,12 @@
 
     from framevitals.cleaning_plan import CleaningPlan
     from framevitals.planning import AnalysisPlan
-    from framevitals.quality_results import DriftResult, GateResult, ValidationResult
+    from framevitals.quality_results import (
+        CheckResult,
+        DriftResult,
+        GateResult,
+        ValidationResult,
+    )
     from framevitals.result import AnalysisResult
 
 
@@ -262,13 +267,20 @@ def check(
     return _check(name, severity=severity, description=description)
 
 
-def run_checks(data: DataInput, checks: Any) -> dict[str, Any]:
+def run_checks(data: DataInput, checks: Any) -> CheckResult:
     """Run exact custom checks through the canonical check engine."""
     from framevitals.checks import run_checks as _run_checks
 
     return _run_checks(data, checks)
 
 
+def discover_checks(*, group: str = "framevitals.checks"):
+    """Explicitly load installed third-party check plugins."""
+    from framevitals.plugins import discover_checks as _discover_checks
+
+    return _discover_checks(group=group)
+
+
 def gate(
     current: DataInput,
     *,
@@ -316,5 +328,6 @@ def gate(
     "validate",
     "check",
     "run_checks",
+    "discover_checks",
     "gate",
 ]

From 0261899de5f86b7d7127a550e460ee34a2e14246 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:32:41 +0530
Subject: [PATCH 288/439] docs: document interoperable source-aware FrameVitals

---
 README.md | 427 +++++++++++++++++++++++++++++++++---------------------
 1 file changed, 261 insertions(+), 166 deletions(-)

diff --git a/README.md b/README.md
index f234f17..6c300bd 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@
 
 ### Know if your data is healthy, stable, and ML-ready — before your model finds out.
 
-**A Python toolkit for data-health diagnostics, drift detection, contracts, quality gates, anomaly analysis, and ML-readiness checks on tabular data.**
+**A source-aware Python toolkit for data health, drift, contracts, quality gates, anomaly analysis, and ML-readiness diagnostics on tabular data.**
 
 [![Tests](https://github.com/parthdongre/FrameVitals/actions/workflows/test.yml/badge.svg)](https://github.com/parthdongre/FrameVitals/actions/workflows/test.yml)
 [![PyPI](https://img.shields.io/pypi/v/framevitals.svg)](https://pypi.org/project/framevitals/)
@@ -12,15 +12,13 @@
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
 [![GitHub stars](https://img.shields.io/github/stars/parthdongre/FrameVitals?style=social)](https://github.com/parthdongre/FrameVitals)
 
-[Install](#installation) · [Quick start](#quick-start) · [Quality gates](#quality-gates) · [CLI](#command-line-interface) · [Roadmap](#roadmap) · [Contributing](#contributing)
+[Install](#installation) · [Quick start](#quick-start) · [Quality gates](#quality-gates) · [Sources](#source-aware-execution) · [CLI](#command-line-interface) · [Roadmap](#roadmap) · [Contributing](#contributing)
 
 </div>
 
 ---
 
-FrameVitals turns a pandas DataFrame or tabular dataset into a **structured health report** that can be inspected, serialized, compared, snapshotted, and enforced in CI.
-
-Instead of stitching together separate profiling, quality, drift, anomaly, contract, and ML-readiness tools, FrameVitals exposes a compact workflow:
+FrameVitals turns a supported tabular source into a **structured health report** that can be inspected, serialized, compared, snapshotted, extended with domain rules, and enforced in CI.
 
 ```python
 import framevitals as fv
@@ -35,68 +33,83 @@ gate = fv.gate(current, reference=reference, contract=contract)
 The goal is simple: **catch bad data before it becomes a bad model, a broken dashboard, or a production incident.**
 
 ```text
-                         ┌──────────────────────────┐
-DataFrame / file ───────►│         ANALYZE          │
-                         │ profile · health · ML     │
-                         │ stats · anomalies · risk  │
-                         └────────────┬─────────────┘
-                                      │
-                                      ▼
-                               structured result
-                                      │
-                        ┌─────────────┴─────────────┐
-                        ▼                           ▼
-                    snapshot                    contract
-                        │                           │
-Reference + current ────┴──► compare / validate ───┘
-                                      │
-                                      ▼
-                                    GATE
-                              PASS / WARN / FAIL
+DataFrame / Arrow / files / DuckDB relation
+                    │
+                    ▼
+              DatasetSource
+                    │
+          ┌─────────┴─────────┐
+          ▼                   ▼
+     bounded stream       exact/full path
+          │                   │
+          └─────────┬─────────┘
+                    ▼
+                 ANALYZE
+                    │
+      ┌─────────────┼─────────────┐
+      ▼             ▼             ▼
+   health        contract      snapshot
+                    │             │
+            ┌───────┘             ▼
+            ▼                  history
+         validate                 │
+            │                     │
+reference ─► compare              │
+            │                     │
+custom ─────┤                     │
+checks      ▼                     │
+           GATE ◄─────────────────┘
+      PASS / WARN / FAIL
 ```
 
 ## Why FrameVitals?
 
-Most data checks answer one narrow question. FrameVitals is designed around the questions that repeatedly show up in real data and ML workflows:
-
 | Question | FrameVitals |
 | --- | --- |
 | Is this dataset structurally healthy? | Missingness, duplicates, cardinality, schema and quality diagnostics |
 | Is it ready for modelling? | ML-readiness scoring, target-aware checks and model diagnostics |
 | Are there suspicious rows or features? | Statistical diagnostics, anomaly detection, leakage and multicollinearity checks |
-| Has production data changed? | Reference-vs-current numeric, categorical and schema drift diagnostics |
-| Can I enforce expectations? | Versioned data contracts, validation and one CI-friendly quality gate |
-| Can I track change without storing every raw dataset? | Compact versioned analysis snapshots and snapshot diffs |
-| Will large Arrow-compatible files be fully loaded by default? | Source-aware bounded execution for supported streaming workflows |
+| Has production data changed? | Numeric, categorical and schema drift diagnostics |
+| Can I enforce expectations? | Versioned contracts, exact validation, custom checks and one quality gate |
+| Can I monitor change without storing every raw batch? | Compact snapshots and persistent snapshot history |
+| Will large supported sources be loaded into pandas by default? | Bounded source-aware execution where semantics permit it |
+| Can I see when FrameVitals sampled or materialized data? | Execution provenance is included in source-aware results |
 | Will analysis unexpectedly write files? | No — filesystem artifacts are opt-in |
 
-FrameVitals is **package-first**. The canonical library lives under `src/framevitals/`; the Flask API and React dashboard are optional interfaces around the same analysis engine.
+FrameVitals is **package-first**. Product logic lives under `src/framevitals/`; the CLI, Flask API, React dashboard and reusable GitHub Action wrap the same canonical engines.
 
 ## Installation
 
-FrameVitals supports **Python 3.11, 3.12, and 3.13**. The current public release is **0.1.0 (alpha)** and is available on PyPI.
+FrameVitals supports **Python 3.11, 3.12, and 3.13**. The current public release is **0.1.0 (alpha)**.
 
 ```bash
 pip install framevitals
 ```
 
-Optional capabilities are installed as extras so the base data-health engine does not need to import every integration stack:
+Optional capabilities are split into extras so the base data-health engine stays focused:
 
 ```bash
-pip install "framevitals[arrow]"  # bounded Parquet / compatible CSV / TSV streaming
-pip install "framevitals[excel]"  # XLS/XLSX readers
-pip install "framevitals[plot]"   # Matplotlib/Seaborn chart and PDF plotting support
-pip install "framevitals[ml]"     # XGBoost, LightGBM, PyOD, SHAP
-pip install "framevitals[ai]"     # Ollama-backed agentic AI features
-pip install "framevitals[web]"    # Flask web runtime
-pip install "framevitals[all]"    # all optional runtime capabilities
+pip install "framevitals[arrow]"   # Parquet + Arrow-backed source interoperability
+pip install "framevitals[duckdb]"  # lazy DuckDB relations + Arrow transport
+pip install "framevitals[excel]"   # XLS/XLSX readers
+pip install "framevitals[plot]"    # Matplotlib/Seaborn charts and report plotting
+pip install "framevitals[ml]"      # XGBoost, LightGBM, PyOD, SHAP
+pip install "framevitals[ai]"      # Ollama-backed AI features
+pip install "framevitals[web]"     # Flask web runtime
+pip install "framevitals[all]"     # all optional runtime capabilities
 ```
 
-Parquet streaming requires the `arrow` extra. With Arrow installed, compatible CSV and TSV inputs can also use the source-aware streaming engine; without Arrow they retain the normal pandas-backed file path. Excel readers are optional through `excel`. Plotting is optional through `plot`, and structured explainability can still fall back to non-plotting importance results when chart support is unavailable.
+On a development checkout:
+
+```bash
+pip install -e ".[all,dev]"
+```
+
+`arrow` enables projection-aware Parquet execution, compatible CSV/TSV streaming, native PyArrow table inputs, and Arrow PyCapsule-compatible table producers. `duckdb` adds lazy `DuckDBPyRelation` inputs without putting DuckDB in the base dependency set.
 
 ## Quick start
 
-### Analyze a DataFrame
+### Analyze a DataFrame or file
 
 ```python
 import pandas as pd
@@ -110,28 +123,64 @@ print(report.ml_readiness)
 print(report.findings[:3])
 ```
 
-File paths work too:
+File paths work directly:
 
 ```python
-report = fv.analyze("customers.csv", mode="quick")
+report = fv.analyze("customers.parquet", mode="quick")
 ```
 
-FrameVitals supports pandas DataFrames and common tabular file formats including CSV, TSV, Excel, JSON, and Parquet when the required reader capability is installed.
+### Analyze Arrow-native data
 
-### Run only the diagnostic you need
+With the `arrow` extra installed:
+
+```python
+import pyarrow as pa
+import framevitals as fv
+
+table = pa.table({
+    "age": [21, 34, 48],
+    "income": [30_000, 62_000, 81_000],
+})
+
+report = fv.analyze(table, mode="quick")
+profile = fv.profile(table)
+```
+
+PyArrow `Table` and `RecordBatch` inputs enter the source-aware batch path instead of being converted to pandas before analysis. Table-like objects implementing the Arrow C Stream / PyCapsule protocol can use the same interoperability boundary when PyArrow is installed.
+
+### Analyze a lazy DuckDB relation
+
+With the `duckdb` extra installed:
+
+```python
+import duckdb
+import framevitals as fv
+
+con = duckdb.connect()
+orders = con.sql("""
+    SELECT *
+    FROM read_parquet('orders/*.parquet')
+""")
 
-The focused API avoids running the complete analysis pipeline when only one answer is required:
+profile = fv.profile(orders)
+report = fv.analyze(orders, mode="quick")
+```
+
+FrameVitals obtains exact relation shape metadata, pushes requested column projection into the relation, and consumes Arrow record batches. It does not call `.df()` for diagnostics that can stay on the streaming path.
+
+### Run only the diagnostic you need
 
 ```python
 profile = fv.profile("customers.parquet")
 health = fv.health("customers.parquet")
 readiness = fv.ml_readiness("customers.parquet")
+quality = fv.quality("customers.parquet")
 stats = fv.statistics("customers.parquet", mode="quick")
 anomalies = fv.anomalies("customers.parquet", mode="quick")
 relationships = fv.relationships("customers.parquet")
 ```
 
-Supported streaming sources use bounded execution where the diagnostic permits it and disclose execution provenance in their result metadata.
+Focused APIs avoid running unrelated pipeline stages. Supported streaming sources use bounded execution where the diagnostic permits it and disclose exact, sampled, estimated, or materialized behavior in result metadata.
 
 ### Add a supervised-learning target
 
@@ -148,42 +197,38 @@ print(report["explainability"])
 
 Target-aware analysis can surface modelling risks such as leakage, imbalance, redundant features, unstable relationships, and weak baselines.
 
+## Drift and contracts
+
 ### Compare datasets for drift
 
 ```python
-reference = pd.read_csv("training_data.csv")
-current = pd.read_csv("production_batch.csv")
-
 result = fv.compare(reference, current)
 
-print(result["summary"]["overall_verdict"])
+print(result.severity)
 print(result["columns"][:3])
+print(result["execution"])
 ```
 
-Numeric drift uses PSI, Kolmogorov-Smirnov statistics, and standardized mean shift. Categorical drift uses PSI and chi-square diagnostics.
+Numeric drift uses PSI, Kolmogorov-Smirnov statistics and standardized mean shift. Categorical drift uses PSI and chi-square diagnostics. Streaming-capable sources can bound distribution work while preserving exact source-shape metadata.
 
-For streaming-capable file sources, value-distribution work is bounded and the result reports the true source shapes plus the sampling strategy used for distribution diagnostics.
-
-### Validate a data contract
-
-Infer a contract once from a trusted reference dataset, then validate later batches before they reach downstream jobs:
+### Infer and validate a contract
 
 ```python
 contract = fv.infer_contract(reference)
 result = fv.validate(current, contract)
 
-if result["status"] == "fail":
-    for finding in result["errors"]:
+if result.status == "fail":
+    for finding in result.findings:
         print(finding["message"])
 ```
 
-Contracts are versioned JSON-friendly dictionaries. Current contract inference can capture required or optional columns, broad data types, nullability and tolerated null rates, finite numeric bounds, low-cardinality allowed values, and uniqueness expectations.
+Contracts are versioned JSON-friendly dictionaries. Current inference can capture required/optional columns, broad data types, nullability, tolerated null rates, finite numeric bounds, low-cardinality allowed values, and uniqueness expectations.
 
-Validation intentionally remains exact for constraints such as uniqueness, allowed values and bounds rather than silently downgrading those checks to sampled approximations.
+**Validation intentionally remains exact.** Constraints such as uniqueness, allowed values and bounds are not silently downgraded to sampled approximations. For a non-pandas source, exact validation may therefore materialize a complete pandas representation, and the `execution.full_materialization` field reports that decision.
 
 ## Quality gates
 
-`fv.gate(...)` combines contract validation and drift into one CI-friendly verdict:
+`fv.gate(...)` combines the checks you choose into one CI-friendly verdict:
 
 ```python
 result = fv.gate(
@@ -198,7 +243,7 @@ print(result.status)   # pass / warn / fail
 print(result.passed)   # True unless the gate failed
 ```
 
-You can run only the checks you need:
+You can run only the families you need:
 
 ```python
 fv.gate(current, contract=contract)
@@ -206,11 +251,49 @@ fv.gate(current, reference=reference)
 fv.gate(current, reference=reference, contract=contract)
 ```
 
-Contract validation remains exact. Drift can use bounded source-aware sampling where a streaming input supports it, and the returned `execution` block records that distinction.
+### Domain-specific custom checks
+
+FrameVitals supports exact application-defined invariants without requiring a fork:
+
+```python
+@fv.check(
+    "positive revenue",
+    severity="error",
+    description="Revenue cannot be negative.",
+)
+def positive_revenue(df):
+    return {
+        "passed": bool((df["revenue"] >= 0).all()),
+        "message": "Negative revenue records were found.",
+    }
+
+checks = fv.run_checks(current, [positive_revenue])
+gate = fv.gate(current, custom_checks=[positive_revenue])
+```
+
+`fv.run_checks()` returns a dict-compatible `CheckResult`. Arbitrary Python checks run against the complete DataFrame because FrameVitals cannot safely infer whether a user-defined invariant is sampleable. Non-pandas sources therefore report full materialization for this exact path.
+
+### Third-party check plugins
+
+Installed packages can register checks through Python entry points under `framevitals.checks`:
+
+```toml
+[project.entry-points."framevitals.checks"]
+positive_revenue = "acme_data_checks:positive_revenue"
+```
+
+Discovery is deliberately **opt-in** because loading an entry point executes provider code:
+
+```python
+checks = fv.discover_checks()
+result = fv.gate(current, custom_checks=checks)
+```
+
+FrameVitals does not import or execute installed check plugins automatically.
 
 ### GitHub Actions
 
-The repository also ships a reusable composite action at the repository root. Once using a release/tag that contains the action, a workflow can gate a dataset without writing custom installation or exit-code glue:
+The repository ships a reusable composite action at `action.yml`:
 
 ```yaml
 - uses: parthdongre/FrameVitals@main
@@ -227,32 +310,41 @@ The repository also ships a reusable composite action at the repository root. On
   run: echo "FrameVitals status: ${{ steps.framevitals.outputs.status }}"
 ```
 
-For production workflows, pin the action to a released tag or commit rather than a moving branch. The action installs the matching FrameVitals source plus the Arrow capability and exposes `status`, `passed`, and `result-path` outputs.
+For production workflows, pin the action to a released tag or commit rather than a moving branch. The action exposes `status`, `passed`, and `result-path` outputs.
 
-## Snapshots
+## Snapshots and monitoring history
 
-An `AnalysisResult` can be reduced to a compact versioned monitoring snapshot:
+An `AnalysisResult` can be reduced to a compact versioned snapshot:
 
 ```python
 report = fv.analyze(current)
 snapshot = report.snapshot("snapshot.json")
 ```
 
-Snapshots retain a fingerprint plus compact state such as schema, missingness, health, ML-readiness and finding codes without embedding the full raw dataset.
+Snapshots retain a fingerprint and compact state such as schema, missingness, health, ML-readiness and finding codes without embedding the full raw dataset.
 
 ```python
 previous = fv.load_snapshot("previous.json")
 latest = fv.load_snapshot("snapshot.json")
 change = fv.compare_snapshots(previous, latest)
+```
+
+For repeated local monitoring, use `SnapshotHistory`:
 
-print(change["schema"])
-print(change["health_delta"])
-print(change["findings"]["new"])
+```python
+history = fv.SnapshotHistory(".framevitals/history")
+
+history.add(report.snapshot(), label="production")
+latest = history.latest()
+previous = history.previous()
+change = history.compare_latest()
 ```
 
+The default `.framevitals/` runtime directory is ignored by Git.
+
 ## The public API
 
-FrameVitals keeps workflow entry points at the package root and exposes focused diagnostics for callers that do not need the full pipeline.
+FrameVitals keeps workflow entry points at the package root while retaining focused diagnostics for callers that need one answer rather than the complete pipeline.
 
 | API | Purpose |
 | --- | --- |
@@ -270,11 +362,15 @@ FrameVitals keeps workflow entry points at the package root and exposes focused
 | `framevitals.compare(...)` | Compare reference/current data for drift |
 | `framevitals.infer_contract(...)` | Infer a reusable versioned data contract |
 | `framevitals.validate(...)` | Validate data against a contract |
-| `framevitals.gate(...)` | Combine validation and drift into one quality verdict |
+| `framevitals.check(...)` | Define a reusable custom invariant |
+| `framevitals.run_checks(...)` | Run exact custom checks and return `CheckResult` |
+| `framevitals.discover_checks(...)` | Explicitly discover installed check plugins |
+| `framevitals.gate(...)` | Combine contract, custom and drift checks into one verdict |
 | `framevitals.create_snapshot(...)` | Create compact monitoring state from an analysis result |
 | `framevitals.compare_snapshots(...)` | Compare two stored analysis states |
+| `framevitals.SnapshotHistory(...)` | Persist and compare a compact local monitoring timeline |
 
-The compatibility module `framevitals.api` delegates lazily to these same canonical engines so older imports do not maintain a second implementation.
+`framevitals.api` remains a lazy compatibility facade over these canonical engines; it does not maintain a second implementation.
 
 ## What FrameVitals checks
 
@@ -282,18 +378,19 @@ The compatibility module `framevitals.api` delegates lazily to these same canoni
 | --- | --- |
 | **Structure** | shape, dtypes, semantic column roles, date/text detection |
 | **Data quality** | missingness, duplicates, constants, cardinality, outliers |
-| **Health scoring** | overall dataset health plus component-level diagnostics |
+| **Health scoring** | overall dataset health plus component diagnostics |
 | **ML readiness** | modelling readiness, risky columns, preprocessing recommendations |
 | **Statistics** | distribution checks, normality, correlations, effect-size style diagnostics |
 | **Anomalies** | multivariate and robust outlier detectors, optional ensemble methods |
 | **Target intelligence** | task inference, leakage hints, multicollinearity, feature/model diagnostics |
-| **Drift** | PSI, KS, chi-square, mean shift, new or disappearing categories |
+| **Drift** | PSI, KS, chi-square, mean shift, new/disappearing categories |
 | **Contracts** | schema, types, nullability, bounds, domains and uniqueness expectations |
+| **Custom invariants** | exact application-specific DataFrame predicates |
 | **Time series** | date-aware diagnostics, stationarity, decomposition and forecast previews |
 | **Text** | text-column profiling, vocabulary and lightweight semantic diagnostics |
-| **Explainability** | model feature importance and SHAP when the optional ML stack is installed |
+| **Explainability** | model feature importance and SHAP when optional capabilities are installed |
 
-Not every analysis runs on every dataset. FrameVitals uses dataset signals, selected mode, target availability, source capabilities, execution budgets, and installed optional dependencies to decide what is useful and safe to execute.
+Not every analysis runs on every dataset. FrameVitals uses dataset signals, selected mode, target availability, source capabilities, execution budgets and installed optional dependencies to decide what is useful and safe to execute.
 
 ## Analysis modes
 
@@ -306,35 +403,52 @@ fv.analyze(data, mode="research")
 
 | Mode | Best for |
 | --- | --- |
-| `quick` | Fast structural, quality, and ML-readiness checks |
+| `quick` | Fast structural, quality and ML-readiness checks |
 | `standard` | Everyday analysis with broader diagnostics |
 | `deep` | Target-aware and heavier statistical analysis |
 | `research` | Largest analysis budget for exploratory work |
 
 ## Source-aware execution
 
-FrameVitals separates a dataset source from the diagnostic that consumes it. That lets supported file formats expose metadata, projection and record batches without forcing every public API to begin with a full pandas materialization.
+FrameVitals separates a **dataset source** from the diagnostic that consumes it. This lets supported inputs expose exact metadata, projection and record batches without forcing every public API to begin with a full pandas conversion.
 
 ```text
-DataFrame / CSV / TSV / Parquet / ...
-                  │
-                  ▼
-            DatasetSource
-                  │
-        ┌─────────┴─────────┐
-        ▼                   ▼
-   streaming path      materialized path
-        │                   │
-        └─────────┬─────────┘
-                  ▼
-         bounded diagnostics
-```
-
-Parquet is projection-aware and streaming through the Arrow capability. Compatible CSV/TSV sources can also use Arrow record batches. Focused diagnostics disclose when a result is exact, derived from a bounded row sample, or required full materialization.
+Pandas DataFrame
+PyArrow Table / RecordBatch
+Arrow C Stream / PyCapsule-compatible table
+CSV / TSV / Parquet
+DuckDB relation
+        │
+        ▼
+   DatasetSource
+        │
+ ┌──────┴─────────┐
+ ▼                ▼
+stream/bounded   exact/full
+ ▼                ▼
+execution provenance
+```
+
+Current source behavior includes:
+
+- **Pandas** — already materialized, canonical in-memory baseline.
+- **Parquet** — Arrow-backed metadata, projection and record batches.
+- **CSV/TSV** — Arrow streaming when compatible and the `arrow` extra is installed; pandas fallback otherwise.
+- **PyArrow Table/RecordBatch** — native in-memory Arrow batch path.
+- **Arrow-compatible table producers** — normalized through the standard Arrow C Stream / PyCapsule boundary when available.
+- **DuckDB relation** — exact count/schema metadata, projection pushed into DuckDB, Arrow batch transport, no `.df()` on streaming-safe diagnostics.
+- **Other supported files** — materialized through the established loader when no streaming adapter exists.
+
+A source being streamable does **not** mean every operation is approximate. FrameVitals chooses execution semantics by diagnostic:
+
+- source shape/schema can remain exact;
+- some statistics, anomaly and drift work can operate on explicit bounded samples;
+- exact contracts and arbitrary custom checks remain full-data operations;
+- every source-aware result should disclose whether data was streamed, sampled, estimated or fully materialized.
 
 ## Filesystem artifacts are opt-in
 
-FrameVitals is designed to behave like a library first. Calling the Python API does not need to scatter reports and cleaned files around the working directory.
+Calling the Python API does not need to scatter reports and cleaned files around the working directory.
 
 ```python
 report = fv.analyze(data)
@@ -346,9 +460,7 @@ print(report["cleaning"]["output_path"])
 
 ## Command-line interface
 
-FrameVitals ships with a CLI for scripts, terminals and CI workflows.
-
-Discover commands and options:
+FrameVitals ships with a CLI for terminals, scripts and CI workflows.
 
 ```bash
 framevitals --help
@@ -358,87 +470,74 @@ framevitals gate --help
 framevitals --version
 ```
 
-Analyze a dataset:
+Analyze and plan:
 
 ```bash
 framevitals analyze dataset.csv
 framevitals analyze dataset.csv --mode quick
 framevitals analyze dataset.csv --target churn --mode deep
 framevitals analyze dataset.csv --output report.json
-framevitals analyze dataset.csv --artifacts
-```
-
-Preview work without executing the heavy stages:
-
-```bash
 framevitals plan dataset.csv --mode standard
 ```
 
-Compare two datasets for drift:
+Compare, contract and gate:
 
 ```bash
-framevitals compare train.csv production.csv
-framevitals compare train.csv production.csv --columns age,income
 framevitals compare train.csv production.csv --fail-on moderate
-framevitals compare train.csv production.csv --output drift.json
-```
-
-Create and validate a contract:
-
-```bash
 framevitals infer-contract training_data.csv --output contract.json
 framevitals validate production_batch.csv --contract contract.json
-```
-
-Run the combined quality gate:
-
-```bash
-framevitals gate production_batch.csv \
-  --reference training_data.csv \
-  --contract contract.json
-```
-
-Tune CI thresholds when needed:
-
-```bash
 framevitals gate production_batch.csv \
   --reference training_data.csv \
   --contract contract.json \
-  --drift-warn-on moderate \
-  --drift-fail-on severe \
-  --fail-on-validation-warning \
   --output gate.json
 ```
 
 CLI exit behavior is intentional:
 
-- `framevitals validate`: `0` for pass/warn by default, `1` for a warning when `--fail-on-warn` is enabled, and `2` for contract failure.
+- `framevitals validate`: `0` for pass/warn by default, `1` for warning when `--fail-on-warn` is enabled, `2` for contract failure.
 - `framevitals compare`: `1` only when `--fail-on` is supplied and the configured drift severity is reached; otherwise `0`.
 - `framevitals gate`: `0` for pass/warn and `1` for fail.
 
 ## Result objects
 
-`fv.analyze(...)` returns an `AnalysisResult`, a dict-compatible result object with notebook- and application-friendly helpers:
+The main workflows return dict-compatible result objects so existing mapping-style code keeps working while application/notebook ergonomics improve:
+
+- `AnalysisResult`
+- `DriftResult`
+- `ValidationResult`
+- `CheckResult`
+- `GateResult`
+- `AnalysisSnapshot`
+
+Example:
 
 ```python
 report = fv.analyze(data)
-
 report.health
-report.ml_readiness
 report.findings
-report.recommendations
 report.column("age")
-report.summary()
 report.to_json("report.json")
 report.to_html("report.html")
 report.snapshot("snapshot.json")
 ```
 
-Drift, validation and gate workflows similarly return dict-compatible `DriftResult`, `ValidationResult` and `GateResult` objects with convenience properties while preserving JSON-friendly mapping behavior during the `0.x` series.
+The `0.x` series preserves room to harden result schemas before `1.0` stability guarantees.
 
 ## Optional capabilities
 
-The default package contains the core data-health engine. Additional integrations are separated into extras and are imported only when their feature is requested.
+The default package contains the core data-health engine. Additional integrations are imported only when their feature is requested.
+
+```bash
+pip install "framevitals[arrow]"
+```
+
+Adds Arrow-backed files and in-memory Arrow interoperability.
+
+```bash
+pip install "framevitals[duckdb]"
+```
+
+Adds lazy DuckDB relation support plus the Arrow transport required by that adapter.
 
 ```bash
 pip install "framevitals[ml]"
@@ -450,13 +549,13 @@ Adds heavier model integrations including XGBoost, LightGBM, PyOD and SHAP.
 pip install "framevitals[ai]"
 ```
 
-Adds Ollama-backed agentic interpretation and question-answering features. AI remains an optional explanation layer; computed diagnostics remain usable without it.
+Adds Ollama-backed interpretation and question-answering features. AI remains an optional explanation layer; computed diagnostics remain usable without it.
 
 ```bash
 pip install "framevitals[plot]"
 ```
 
-Adds Matplotlib/Seaborn-backed chart rendering and PDF/report plotting. Structured diagnostics do not require this extra.
+Adds Matplotlib/Seaborn-backed chart rendering and report plotting. Structured diagnostics do not require this extra.
 
 ```bash
 pip install "framevitals[excel]"
@@ -481,28 +580,25 @@ npm ci
 npm run dev
 ```
 
-The `web` extra contains the server runtime only. Install `plot` as well for server-side chart/PDF artifacts and `ai` for the agentic Q&A path. Missing optional AI/report capabilities are loaded lazily instead of preventing the web server from starting.
+The `web` extra contains the server runtime only. Install `plot` for server-side chart/report artifacts and `ai` for the agentic Q&A path. Optional stacks are loaded lazily rather than blocking Flask startup.
 
 Typical local endpoints:
 
 - Flask API: `http://127.0.0.1:5055`
 - React dashboard: `http://127.0.0.1:5173`
 
-The public project website can remain separate from the package runtime so the Python library does not depend on a hosted service.
-
 ## Design principles
 
-FrameVitals is being built around a few constraints that are easy to lose in analytics projects:
-
-- **Package first** — the Python API is canonical; web interfaces wrap it rather than define it.
+- **Package first** — Python APIs are canonical; interfaces wrap them.
 - **Structured results** — return reusable data, not only screenshots or prose.
-- **Source aware** — inspect and stream supported sources instead of eagerly loading everything by default.
+- **Source aware** — inspect and stream supported inputs before choosing materialization.
 - **Bounded execution** — expensive diagnostics operate within explicit scale budgets.
-- **Transparent approximations** — sampled or approximate results disclose their execution provenance.
-- **Exact where correctness requires it** — contract constraints are not silently weakened to sampled checks.
-- **Safe defaults** — no unexpected artifact writes and graceful optional-feature fallbacks.
-- **Small workflow surface** — make analyze → compare → validate → gate obvious while keeping focused diagnostics available.
-- **Optional integrations** — Arrow, Excel, plotting, ML, AI, and web capabilities can be installed independently.
+- **Transparent approximations** — sampled or estimated results disclose provenance.
+- **Exact where correctness requires it** — contracts and arbitrary custom checks are not silently weakened.
+- **Safe defaults** — no unexpected artifact writes or automatic plugin execution.
+- **Small workflow surface** — analyze → compare → validate → gate → monitor stays obvious.
+- **Optional integrations** — Arrow, DuckDB, Excel, plotting, ML, AI and web are independently installable.
+- **Extensible without forks** — domain checks and opt-in entry-point plugins can extend the gate.
 
 ## Project layout
 
@@ -515,10 +611,11 @@ FrameVitals is being built around a few constraints that are easy to lose in ana
 ├── frontend/                 # optional React + TypeScript dashboard
 ├── templates/                # Flask report pages
 ├── static/                   # web/report assets
+├── examples/                 # focused usage examples
 ├── action.yml                # reusable GitHub Actions quality gate
 ├── app.py                    # optional Flask API/server
 ├── pyproject.toml            # package metadata and dependency groups
-└── .github/workflows/        # CI, package, action, benchmark and publishing workflows
+└── .github/workflows/        # CI, interop, package, benchmark and release workflows
 ```
 
 New reusable Python code belongs in `src/framevitals/` and should import through the `framevitals.*` namespace.
@@ -546,7 +643,7 @@ On Windows PowerShell:
 .venv\Scripts\Activate.ps1
 ```
 
-CI validates the core package across Python 3.11–3.13, Arrow streaming paths, optional dependency boundaries, the reusable Gate Action, the Rust workspace/native bridge, React build, wheel contents, distribution metadata, release-version consistency, and a clean-wheel install. A separate benchmark workflow records reproducible profile time and peak-RSS measurements on `main` or when manually dispatched.
+CI validates the core package across Python 3.11–3.13, Arrow streaming paths, Arrow/DuckDB interoperability, optional dependency boundaries, the reusable Gate Action, the Rust workspace/native bridge, React build, wheel contents, distribution metadata, release-version consistency, and clean-wheel installation. A separate benchmark workflow records reproducible time and peak-RSS measurements.
 
 Development is integrated through `dev`; `main` is kept release-ready.
 
@@ -559,33 +656,31 @@ FrameVitals is moving toward a dependable data-health quality gate rather than a
      analyze · focused diagnostics · drift · package/CLI baseline
 
 0.2  SOURCE-AWARE QUALITY GATES
-     bounded streaming · contracts · validate · gate · snapshots · execution provenance
+     bounded streaming · contracts · validate · gate · snapshots · provenance
 
 0.3  STABLE RESULTS + PERFORMANCE
-     result-schema hardening · performance regression CI · lighter installs · richer history
+     result-schema hardening · regression budgets · lighter installs · richer history
 
-0.4  EXTENSIBILITY + DATA ADAPTERS
-     custom checks · plugin surface · Arrow/Polars/DuckDB-style adapters · integrations
+0.4  EXTENSIBILITY + INTEROPERABILITY
+     custom checks · entry-point plugins · source protocols · integrations
 
 1.0  STABLE DATA-HEALTH API
      dependable analyze → compare → validate → gate → monitor workflow
 ```
 
-Several `0.2` capabilities are already under active development on the repository development branches. Before `1.0`, naming, thresholds, result schemas and extension points may still evolve.
+Several post-0.1 capabilities are already under active development on repository development branches. Before `1.0`, naming, thresholds, result schemas and extension points may still evolve.
 
 ## Project status
 
-FrameVitals `0.1.x` is **alpha software**. The project already has a usable package API, CLI, contracts, gates, snapshots, source-aware execution, tests and release tooling, but the `0.x` series deliberately leaves room to refine result schemas, thresholds, dependency boundaries and extension points before stability guarantees begin.
+FrameVitals `0.1.x` is **alpha software**. The project already has a package API, CLI, contracts, gates, snapshots, source-aware execution, tests and release tooling, while the `0.x` series deliberately leaves room to refine schemas, thresholds, dependency boundaries and extension points before stability guarantees begin.
 
 Feedback on real datasets, false positives, missing diagnostics, performance, source compatibility and API ergonomics is especially valuable.
 
 ## Contributing
 
-Contributions are welcome.
-
-A good contribution is focused, tested, and improves either the reliability of a diagnostic, the clarity of the public workflow, source compatibility, or execution transparency.
+Contributions are welcome. A good contribution is focused, tested, and improves diagnostic reliability, public workflow clarity, source compatibility, execution transparency or extension ergonomics.
 
-Start with [CONTRIBUTING.md](CONTRIBUTING.md), and please read the [Code of Conduct](CODE_OF_CONDUCT.md) and [Security Policy](SECURITY.md).
+Start with [CONTRIBUTING.md](CONTRIBUTING.md), and read the [Code of Conduct](CODE_OF_CONDUCT.md) and [Security Policy](SECURITY.md).
 
 ## Releases
 

From bc22b98abb28299549e5ecf346089da3cbbc54fe Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:33:05 +0530
Subject: [PATCH 289/439] docs: record interoperability and extensibility work

---
 CHANGELOG.md | 36 +++++++++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 13fdb40..b26f135 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,56 +12,70 @@ Development continues on the development branches. Changes intended for the next
 
 - Data-contract inference through `framevitals.infer_contract()` with versioned contract schemas, tolerant null-rate and numeric-bound expectations, categorical domains, optional-column semantics, and uniqueness hints
 - Structured exact contract validation through `framevitals.validate()`
-- `framevitals.gate()` for one CI-friendly pass/warn/fail verdict combining validation and drift
+- `framevitals.gate()` for one CI-friendly pass/warn/fail verdict combining contract validation, drift, and optional custom checks
 - `framevitals gate` CLI command with configurable drift warning/failure thresholds, optional contract enforcement, JSON output, and CI exit codes
 - Reusable root `action.yml` GitHub Action for running FrameVitals quality gates with status/result outputs
 - Path-scoped smoke CI for the reusable Gate Action
-- `framevitals plan` for previewing applicable modules, execution budgets, and source planning without running the heavy pipeline
+- `framevitals plan` for previewing applicable modules, execution budgets, source capabilities, and bounded planning samples
 - Compact versioned analysis snapshots with deterministic fingerprints, JSON persistence, and snapshot-to-snapshot schema/health/ML-readiness/finding diffs
-- Dict-compatible result objects including `AnalysisResult`, `DriftResult`, `ValidationResult`, `GateResult`, and per-column result views
+- `SnapshotHistory` for persistent compact monitoring timelines, latest/previous lookup, labels, and recent-state comparison without storing raw datasets
+- Dict-compatible result objects including `AnalysisResult`, `DriftResult`, `ValidationResult`, `CheckResult`, `GateResult`, and per-column result views
+- `framevitals.check()` and `framevitals.run_checks()` for exact user-defined DataFrame invariants with warning/error severities and structured findings
+- Opt-in `framevitals.discover_checks()` plugin discovery through the `framevitals.checks` Python entry-point group
 - Focused public APIs for profiling, column roles, health, ML readiness, quality, statistics, anomalies, relationships, and target analysis without running unrelated pipeline stages
 - Source abstraction with metadata inspection, projection, and streaming capability discovery
 - Bounded Arrow-backed Parquet execution for public profiling, health, ML-readiness, roles, quality, statistics, anomaly, relationship, target-aware, drift, gate, planning, and full-analysis paths where those diagnostics support bounded execution
 - Optional Arrow-backed CSV and TSV streaming with exact row-count metadata, projected record batches, and pandas fallback for incompatible inputs
+- Native PyArrow `Table` and `RecordBatch` inputs through the same source-aware streaming engine
+- Arrow C Stream / PyCapsule-compatible table input support without requiring a library-specific adapter for every producer
+- Optional lazy DuckDB relation source with exact count/schema metadata, projection pushdown, Arrow record batches, and full pandas materialization only for exact/full-row APIs
+- Optional `duckdb` dependency group containing DuckDB plus its Arrow transport
+- Dedicated Arrow/DuckDB interoperability CI coverage
 - Execution provenance that distinguishes exact full-stream metrics, bounded row samples, estimates, and full materialization
 - Optional Rust native core and Python bridge for accelerated streaming numeric state, cardinality/quantile sketches, and backend routing
 - Dedicated Arrow/NumPy-fallback and Rust/native CI coverage
 - Reproducible performance benchmark workflow with time/RSS summaries and retained JSON artifacts
 - PEP 561 `py.typed` marker for downstream type-checking support
 - Optional `excel` dependency group for XLS/XLSX readers
-- Optional `plot` dependency group for Matplotlib/Seaborn chart and PDF/report support
+- Optional `plot` dependency group for Matplotlib/Seaborn chart and report plotting support
 - Contributor architecture guidance covering canonical layers, streaming provenance, optional-dependency boundaries, and performance-sensitive changes
+- Focused examples for domain-specific quality gates and persistent snapshot monitoring
 
 ### Changed
 
-- Full `framevitals.analyze()` now dispatches streaming-capable sources through a bounded source-aware pipeline when artifacts do not require materialization
+- Full `framevitals.analyze()` now resolves generic `DatasetSource` inputs and dispatches streaming-capable sources through a bounded source-aware pipeline when artifacts do not require materialization
 - Drift comparison now preserves exact source shape metadata while bounding value-distribution work for streaming sources
 - Flask `/api/compare` now routes through the same canonical source-aware drift engine as Python and CLI callers
-- `framevitals.gate()` reuses the same canonical drift and validation engines instead of maintaining separate data-loading logic
+- `framevitals.gate()` reuses the same canonical drift, validation, and custom-check engines instead of maintaining separate data-loading logic
 - Contract validation explicitly remains exact for uniqueness, allowed-value, nullability, and bound constraints rather than silently weakening them to sampled checks
-- `framevitals.api` is now a lazy compatibility facade over the canonical focused, analysis, planning, and operations engines instead of a second eager implementation
+- Arbitrary custom Python checks explicitly remain exact and materialize non-pandas sources rather than silently sampling user-defined invariants
+- Top-level gate execution now reports whether any selected check family required full materialization
+- Materialization provenance is now based on actual execution semantics rather than assuming only file-backed sources can materialize
+- `framevitals.api` is now a lazy compatibility facade over the canonical focused, analysis, planning, operations, check, and plugin engines instead of a second eager implementation
 - The installed CLI now routes directly through canonical APIs and no longer relies on a runtime monkeypatch shim
 - CLI `--version` works from a no-dependencies wheel smoke install without importing the analytics stack
-- The `all` extra now actually includes the Arrow capability
+- The `all` extra now includes Arrow and DuckDB interoperability capabilities
 - Excel engines moved out of the default dependency set into `framevitals[excel]`; AI-only Pydantic moved into `framevitals[ai]`
 - Matplotlib and Seaborn moved out of the default dependency set into `framevitals[plot]`
 - Explainability no longer requires Matplotlib at module import time; structured importance can run without plotting and SHAP chart rendering is optional
-- Flask startup no longer eagerly imports the agentic AI or PDF/report stacks; those capabilities load only when their endpoints are used
+- Flask startup no longer eagerly imports the agentic AI or report/plot stacks; those capabilities load only when their endpoints are used
 - Excel loading now reports a FrameVitals-specific install hint when the optional reader capability is missing
-- README and roadmap now describe the source-aware analyze → compare → validate → gate → monitor direction, optional capability groups, reusable Gate Action, and actual CLI exit semantics
+- README and roadmap now describe the source-aware analyze → compare → validate → gate → monitor direction, source interoperability, exact-vs-bounded execution, optional capability groups, plugins, and the reusable Gate Action
 - Package CI now requires the `py.typed` marker to be present in built wheels
 - Test and package workflows now use read-only default permissions and cancel stale runs on the same branch
 - Release publishing now verifies that the GitHub release tag, `pyproject.toml` version, and `framevitals.__version__` match before building/publishing
 
 ### Fixed
 
+- CSV/TSV no-Arrow fallback no longer relies on zero-argument `super()` in a slotted dataclass subclass, restoring Python 3.11/3.12 compatibility
 - Streaming ML-readiness results now disclose when duplicate rate is estimated from a bounded sample instead of reporting sampled data as exact
 - Streaming relationship metadata reports true source size instead of sample size
 - Streaming column-role cardinality metadata distinguishes bounded-sample cardinality from full-stream native estimates
 - CSV/TSV streaming sources preserve `FileSource` compatibility for existing callers and tests
-- Dedicated Arrow CI now exercises streaming statistics, drift, CSV/TSV sources, and the quality-gate CLI
+- Dedicated Arrow CI now exercises streaming statistics, drift, CSV/TSV sources, in-memory Arrow inputs, and the quality-gate CLI
 - Web runtime dependency boundaries no longer force AI/report/plot packages to import during Flask startup
 - Reusable Gate Action setup no longer relies on an invalid absolute `setup-python` cache dependency path
+- Exact validation/custom-check provenance now reports full materialization for Arrow and relation-backed inputs when they are converted to complete pandas DataFrames
 
 ## 0.1.0 - 2026-08-15
 

From 2dfa5b74c395d5f5aefc8111a12f2e6351889657 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:33:47 +0530
Subject: [PATCH 290/439] feat: add public source capability inspection

---
 src/framevitals/sources.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/framevitals/sources.py b/src/framevitals/sources.py
index b4e1f39..ee146bf 100644
--- a/src/framevitals/sources.py
+++ b/src/framevitals/sources.py
@@ -420,3 +420,8 @@ def resolve_source(data: Any) -> DatasetSource:
         "data must be a pandas DataFrame, Arrow-compatible table, DuckDB relation, "
         "dataset path, or DatasetSource."
     )
+
+
+def inspect_source(data: Any) -> dict[str, Any]:
+    """Return source metadata/capabilities without running analysis diagnostics."""
+    return resolve_source(data).inspect().to_dict()

From 27d247b34d6afc15fe53d95bb369d5a197c6d32c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:34:15 +0530
Subject: [PATCH 291/439] feat: expose source inspection at package root

---
 src/framevitals/__init__.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 4e6193a..4ee7234 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -41,6 +41,13 @@ def __getattr__(name: str):
     raise AttributeError(name)
 
 
+def inspect_source(data: Any) -> dict[str, Any]:
+    """Inspect source metadata and execution capabilities without analysis."""
+    from framevitals.sources import inspect_source as _inspect_source
+
+    return _inspect_source(data)
+
+
 def profile(data: Any) -> dict[str, Any]:
     """Profile shape, dtypes, missingness, cardinality, and basic summaries."""
     from framevitals.focused import profile as _profile
@@ -341,6 +348,7 @@ def gate(
     "DriftResult",
     "GateResult",
     "ValidationResult",
+    "inspect_source",
     "profile",
     "roles",
     "health",

From 5dfd74e0a738be0d99fbee309451c20e7851217d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:34:38 +0530
Subject: [PATCH 292/439] feat: mirror source inspection in compatibility API

---
 src/framevitals/api.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index 99f7b80..2b2dee4 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -29,6 +29,13 @@
 DataInput = Any
 
 
+def inspect_source(data: DataInput) -> dict[str, Any]:
+    """Inspect source metadata/capabilities without running analysis."""
+    from framevitals.sources import inspect_source as _inspect_source
+
+    return _inspect_source(data)
+
+
 def profile(data: DataInput) -> dict[str, Any]:
     """Profile a dataset through the canonical focused execution path."""
     from framevitals.focused import profile as _profile
@@ -310,6 +317,7 @@ def gate(
 
 
 __all__ = [
+    "inspect_source",
     "profile",
     "roles",
     "health",

From 54e2ae19fb66855cdb229ed5d3e3baf55365b98a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:34:54 +0530
Subject: [PATCH 293/439] test: cover public source capability inspection

---
 tests/test_source_inspection.py | 80 +++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 tests/test_source_inspection.py

diff --git a/tests/test_source_inspection.py b/tests/test_source_inspection.py
new file mode 100644
index 0000000..e9d076b
--- /dev/null
+++ b/tests/test_source_inspection.py
@@ -0,0 +1,80 @@
+import pandas as pd
+import pytest
+
+import framevitals as fv
+from framevitals.sources import DatasetMetadata, DelimitedTextSource
+
+
+def test_inspect_source_reports_pandas_capabilities():
+    frame = pd.DataFrame({
+        "value": [1, 2, 3],
+        "label": ["a", "b", "c"],
+    })
+
+    info = fv.inspect_source(frame)
+
+    assert info["name"] == "<dataframe>"
+    assert info["kind"] == "memory"
+    assert info["format"] == "pandas"
+    assert info["rows"] == 3
+    assert info["columns"] == 2
+    assert info["size_bytes"] > 0
+    assert info["materialized"] is True
+    assert info["supports_projection"] is True
+    assert info["supports_streaming"] is False
+
+
+def test_inspect_source_reports_csv_fallback_without_arrow(tmp_path, monkeypatch):
+    path = tmp_path / "data.csv"
+    pd.DataFrame({"value": [1, 2, 3]}).to_csv(path, index=False)
+    monkeypatch.setattr(DelimitedTextSource, "_pyarrow_csv", lambda self: None)
+
+    info = fv.inspect_source(path)
+
+    assert info["name"] == "data.csv"
+    assert info["kind"] == "file"
+    assert info["format"] == "csv"
+    assert info["rows"] is None
+    assert info["columns"] is None
+    assert info["size_bytes"] == path.stat().st_size
+    assert info["materialized"] is False
+    assert info["supports_projection"] is False
+    assert info["supports_streaming"] is False
+
+
+def test_inspect_source_accepts_custom_dataset_source():
+    class CustomSource:
+        def inspect(self):
+            return DatasetMetadata(
+                name="custom",
+                kind="remote",
+                format="custom",
+                rows=42,
+                columns=4,
+                size_bytes=None,
+                materialized=False,
+                supports_projection=True,
+                supports_streaming=False,
+            )
+
+        def load(self):
+            return pd.DataFrame({"value": [1]})
+
+    info = fv.inspect_source(CustomSource())
+
+    assert info == {
+        "name": "custom",
+        "kind": "remote",
+        "format": "custom",
+        "rows": 42,
+        "columns": 4,
+        "size_bytes": None,
+        "materialized": False,
+        "supports_projection": True,
+        "supports_streaming": False,
+    }
+
+
+def test_inspect_source_rejects_unsupported_input():
+    with pytest.raises(TypeError, match="dataset path"):
+        fv.inspect_source(object())

From 57fd09e728cfb2036de10b15d7cf50a13e3752cb Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:35:13 +0530
Subject: [PATCH 294/439] test: expose Arrow source capabilities publicly

---
 tests/test_arrow_memory.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/test_arrow_memory.py b/tests/test_arrow_memory.py
index a03396b..38dffa1 100644
--- a/tests/test_arrow_memory.py
+++ b/tests/test_arrow_memory.py
@@ -34,6 +34,9 @@ def test_arrow_table_source_exposes_exact_metadata_projection_and_batches():
     assert metadata.supports_streaming is True
     assert source.schema() == table.schema
 
+    public_info = framevitals.inspect_source(table)
+    assert public_info == metadata.to_dict()
+
     batches = list(source.iter_batches(batch_size=700, columns=["value", "group"]))
     assert sum(batch.num_rows for batch in batches) == table.num_rows
     assert max(batch.num_rows for batch in batches) <= 700
@@ -112,6 +115,11 @@ def __arrow_c_stream__(self, requested_schema=None):
     assert metadata.columns == table.num_columns
     assert metadata.supports_streaming is True
 
+    public_info = framevitals.inspect_source(CapsuleTable())
+    assert public_info["format"] == "arrow"
+    assert public_info["rows"] == table.num_rows
+    assert public_info["supports_streaming"] is True
+
     result = framevitals.profile(CapsuleTable())
     assert result["shape"] == {"rows": table.num_rows, "columns": table.num_columns}
     assert result["source_metadata"]["format"] == "arrow"

From b928c2a4dcfceb7946408a524d6cf28eea73978e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:35:32 +0530
Subject: [PATCH 295/439] test: expose DuckDB source capabilities publicly

---
 tests/test_duckdb_source.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_duckdb_source.py b/tests/test_duckdb_source.py
index 5a1a839..dc6909e 100644
--- a/tests/test_duckdb_source.py
+++ b/tests/test_duckdb_source.py
@@ -49,6 +49,9 @@ def test_duckdb_relation_source_exposes_exact_metadata_projection_and_batches():
         assert metadata.supports_streaming is True
         assert source.schema().names == ["value", "other", "grp"]
 
+        public_info = framevitals.inspect_source(relation)
+        assert public_info == metadata.to_dict()
+
         batches = list(source.iter_batches(batch_size=700, columns=["value", "grp"]))
         assert sum(batch.num_rows for batch in batches) == 4_000
         assert max(batch.num_rows for batch in batches) <= 700

From dd16918e339e11dc8af15e371cc2db129cfc8f0e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:36:33 +0530
Subject: [PATCH 296/439] feat: define execution provenance schema

---
 src/framevitals/provenance.py | 102 ++++++++++++++++++++++++++++++++++
 1 file changed, 102 insertions(+)
 create mode 100644 src/framevitals/provenance.py

diff --git a/src/framevitals/provenance.py b/src/framevitals/provenance.py
new file mode 100644
index 0000000..16db8c9
--- /dev/null
+++ b/src/framevitals/provenance.py
@@ -0,0 +1,102 @@
+"""Shared execution-provenance helpers for public FrameVitals results.
+
+The 0.x API historically grew execution metadata inside individual diagnostics.
+This module defines a small additive contract so those payloads can converge
+without breaking callers that still rely on legacy fields.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+
+EXECUTION_SCHEMA_VERSION = "1"
+
+
+def load_fully_materializes(metadata: Any) -> bool:
+    """Return whether ``source.load()`` creates a complete pandas representation.
+
+    A pandas DataFrame is already materialized before FrameVitals sees it. File,
+    Arrow, relation, remote, and other source types may all require a complete
+    conversion when an exact operation calls ``load()``; source kind alone is
+    therefore not a reliable materialization signal.
+    """
+    return not (
+        getattr(metadata, "kind", None) == "memory"
+        and getattr(metadata, "format", None) == "pandas"
+    )
+
+
+def execution_provenance(
+    method: str,
+    *,
+    full_materialization: bool,
+    source: Mapping[str, Any] | None = None,
+    sampled: bool | None = None,
+    source_rows: int | None = None,
+    source_columns: int | None = None,
+    sample_rows: int | None = None,
+    strategy: str | None = None,
+    components: Mapping[str, Any] | None = None,
+    reason: str | None = None,
+    scope: str | None = None,
+    extra: Mapping[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Build a JSON-friendly execution block using the shared v1 vocabulary.
+
+    Optional values are omitted rather than serialized as ``null`` so existing
+    result payloads can adopt the schema incrementally. ``extra`` exists for
+    operation-specific fields such as scale class, pair budget, or projection
+    counts without making them part of the universal contract.
+    """
+    if not str(method).strip():
+        raise ValueError("execution provenance method must not be empty.")
+
+    payload: dict[str, Any] = {
+        "execution_schema_version": EXECUTION_SCHEMA_VERSION,
+        "method": str(method),
+        "full_materialization": bool(full_materialization),
+    }
+    optional = {
+        "source": dict(source) if source is not None else None,
+        "sampled": bool(sampled) if sampled is not None else None,
+        "source_rows": int(source_rows) if source_rows is not None else None,
+        "source_columns": int(source_columns) if source_columns is not None else None,
+        "sample_rows": int(sample_rows) if sample_rows is not None else None,
+        "strategy": strategy,
+        "components": dict(components) if components is not None else None,
+        "reason": reason,
+        "scope": scope,
+    }
+    payload.update({key: value for key, value in optional.items() if value is not None})
+    if extra:
+        payload.update(dict(extra))
+    return payload
+
+
+def normalize_execution(
+    execution: Mapping[str, Any],
+    *,
+    method: str | None = None,
+    full_materialization: bool | None = None,
+    source: Mapping[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Add the shared v1 contract to an existing legacy execution mapping.
+
+    Existing keys always win unless the caller explicitly supplies a method,
+    materialization flag, or source. This makes migration additive during 0.x.
+    """
+    result = dict(execution)
+    result["execution_schema_version"] = EXECUTION_SCHEMA_VERSION
+
+    resolved_method = method or result.get("method") or result.get("scope")
+    if resolved_method is not None:
+        result["method"] = str(resolved_method)
+    if full_materialization is not None:
+        result["full_materialization"] = bool(full_materialization)
+    else:
+        result.setdefault("full_materialization", False)
+    if source is not None:
+        result["source"] = dict(source)
+    return result

From 547dd6e61a1b088aae3835798ab94d8322ddbce4 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:37:03 +0530
Subject: [PATCH 297/439] refactor: standardize custom check execution
 provenance

---
 src/framevitals/checks.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/src/framevitals/checks.py b/src/framevitals/checks.py
index 0c8348f..f8b1daf 100644
--- a/src/framevitals/checks.py
+++ b/src/framevitals/checks.py
@@ -10,6 +10,7 @@
 import numpy as np
 import pandas as pd
 
+from framevitals.provenance import execution_provenance, load_fully_materializes
 from framevitals.quality_results import CheckResult
 from framevitals.sources import resolve_source
 
@@ -30,11 +31,6 @@ def _validate_severity(value: str) -> CheckSeverity:
     return cast(CheckSeverity, value)
 
 
-def _load_fully_materializes_source(metadata) -> bool:
-    """Whether exact custom checks create a complete pandas representation."""
-    return not (metadata.kind == "memory" and metadata.format == "pandas")
-
-
 @dataclass(frozen=True, slots=True)
 class DataCheck:
     """Named user-defined data check with gate severity metadata."""
@@ -189,6 +185,19 @@ def run_checks(
         for result in failures
     ]
 
+    execution = execution_provenance(
+        "exact_custom_checks",
+        full_materialization=load_fully_materializes(metadata),
+        source=metadata.to_dict(),
+        sampled=False,
+        source_rows=metadata.rows,
+        source_columns=metadata.columns,
+        reason=(
+            "Arbitrary custom Python checks run on the complete DataFrame; "
+            "FrameVitals does not silently sample user-defined invariants."
+        ),
+    )
+
     return CheckResult({
         "status": status,
         "passed": status != "fail",
@@ -200,13 +209,5 @@ def run_checks(
             "warnings": len(warning_failures),
             "errors": len(error_failures),
         },
-        "execution": {
-            "method": "exact_custom_checks",
-            "full_materialization": _load_fully_materializes_source(metadata),
-            "source": metadata.to_dict(),
-            "reason": (
-                "Arbitrary custom Python checks run on the complete DataFrame; "
-                "FrameVitals does not silently sample user-defined invariants."
-            ),
-        },
+        "execution": execution,
     })

From abcaa3082a684a03aee570388cfe7f7ed5218912 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:37:39 +0530
Subject: [PATCH 298/439] refactor: standardize operation execution provenance

---
 src/framevitals/operations.py | 141 ++++++++++++++++++++--------------
 1 file changed, 85 insertions(+), 56 deletions(-)

diff --git a/src/framevitals/operations.py b/src/framevitals/operations.py
index a3d74cb..defb7f9 100644
--- a/src/framevitals/operations.py
+++ b/src/framevitals/operations.py
@@ -22,6 +22,7 @@
 from framevitals.contracts import validate_contract
 from framevitals.drift_analysis import compare_datasets, severity_at_least
 from framevitals.profiler import build_profile
+from framevitals.provenance import execution_provenance, load_fully_materializes
 from framevitals.quality_results import DriftResult, GateResult, ValidationResult
 from framevitals.sources import DatasetMetadata, StreamingDatasetSource, resolve_source
 
@@ -58,11 +59,6 @@ def _load_input(value: DataInput, *, label: str) -> tuple[pd.DataFrame, str]:
     return dataframe, metadata.name
 
 
-def _load_fully_materializes_source(metadata: DatasetMetadata) -> bool:
-    """Whether ``source.load()`` creates a complete pandas representation."""
-    return not (metadata.kind == "memory" and metadata.format == "pandas")
-
-
 def _comparison_frame(
     source,
     metadata: DatasetMetadata,
@@ -78,32 +74,37 @@ def _comparison_frame(
         sample = sample_streaming_source(source, sample_rows=sample_rows)
         source_rows = int(metadata.rows)
         sampled = len(sample) < source_rows
-        return sample, {
-            "source_rows": source_rows,
-            "source_columns": int(metadata.columns or len(sample.columns)),
-            "sample_rows": int(len(sample)),
-            "sampled": sampled,
-            "strategy": (
-                "streaming_evenly_spaced_global_rows"
-                if sampled
-                else "full_stream_via_batches"
-            ),
-            "full_materialization": False,
-            "source": metadata.to_dict(),
-        }
+        strategy = (
+            "streaming_evenly_spaced_global_rows"
+            if sampled
+            else "full_stream_via_batches"
+        )
+        execution = execution_provenance(
+            "streaming_source_compare_input",
+            full_materialization=False,
+            source=metadata.to_dict(),
+            sampled=sampled,
+            source_rows=source_rows,
+            source_columns=int(metadata.columns or len(sample.columns)),
+            sample_rows=int(len(sample)),
+            strategy=strategy,
+        )
+        return sample, execution
 
     dataframe = source.load()
     if dataframe.empty:
         raise ValueError(f"Dataset is empty: {metadata.name}")
-    return dataframe, {
-        "source_rows": int(len(dataframe)),
-        "source_columns": int(len(dataframe.columns)),
-        "sample_rows": int(len(dataframe)),
-        "sampled": False,
-        "strategy": "full_input",
-        "full_materialization": _load_fully_materializes_source(metadata),
-        "source": metadata.to_dict(),
-    }
+    execution = execution_provenance(
+        "full_compare_input",
+        full_materialization=load_fully_materializes(metadata),
+        source=metadata.to_dict(),
+        sampled=False,
+        source_rows=int(len(dataframe)),
+        source_columns=int(len(dataframe.columns)),
+        sample_rows=int(len(dataframe)),
+        strategy="full_input",
+    )
+    return dataframe, execution
 
 
 def _true_shape(execution: Mapping[str, Any]) -> list[int]:
@@ -177,24 +178,26 @@ def compare(
     any_sampled = bool(
         reference_execution["sampled"] or current_execution["sampled"]
     )
-    payload["execution"] = {
-        "method": "bounded_source_compare" if any_sampled else "full_compare",
-        "sample_limit_rows_per_source": _DRIFT_SAMPLE_ROWS,
-        "full_materialization": bool(
+    components = {
+        "source_shape": "exact",
+        "schema_columns": "exact",
+        "value_distributions": "bounded_row_sample" if any_sampled else "full_input",
+        "missingness": "bounded_row_sample" if any_sampled else "full_input",
+    }
+    payload["execution"] = execution_provenance(
+        "bounded_source_compare" if any_sampled else "full_compare",
+        full_materialization=bool(
             reference_execution["full_materialization"]
             or current_execution["full_materialization"]
         ),
-        "reference": reference_execution,
-        "current": current_execution,
-        "components": {
-            "source_shape": "exact",
-            "schema_columns": "exact",
-            "value_distributions": (
-                "bounded_row_sample" if any_sampled else "full_input"
-            ),
-            "missingness": "bounded_row_sample" if any_sampled else "full_input",
+        sampled=any_sampled,
+        components=components,
+        extra={
+            "sample_limit_rows_per_source": _DRIFT_SAMPLE_ROWS,
+            "reference": reference_execution,
+            "current": current_execution,
         },
-    }
+    )
     return DriftResult(payload)
 
 
@@ -234,15 +237,18 @@ def validate(
         raise ValueError(f"Dataset is empty: {metadata.name}")
     payload = validate_contract(dataframe, contract)
     payload["dataset_name"] = metadata.name
-    payload["execution"] = {
-        "method": "exact_contract_validation",
-        "full_materialization": _load_fully_materializes_source(metadata),
-        "source": metadata.to_dict(),
-        "reason": (
+    payload["execution"] = execution_provenance(
+        "exact_contract_validation",
+        full_materialization=load_fully_materializes(metadata),
+        source=metadata.to_dict(),
+        sampled=False,
+        source_rows=metadata.rows,
+        source_columns=metadata.columns,
+        reason=(
             "Contract validation remains exact; uniqueness, allowed-value, and bound "
             "constraints are not silently downgraded to sampled checks."
         ),
-    }
+    )
     return ValidationResult(payload)
 
 
@@ -387,6 +393,37 @@ def fail() -> None:
         for block in (validation_execution, drift_execution, custom_execution)
         if isinstance(block, Mapping)
     ]
+    full_materialization = any(
+        bool(block.get("full_materialization")) for block in execution_blocks
+    )
+
+    gate_execution = execution_provenance(
+        "quality_gate",
+        full_materialization=full_materialization,
+        source=current_metadata.to_dict(),
+        components={
+            "validation": (
+                validation_execution.get("method")
+                if isinstance(validation_execution, Mapping)
+                else None
+            ),
+            "drift": (
+                drift_execution.get("method")
+                if isinstance(drift_execution, Mapping)
+                else None
+            ),
+            "custom": (
+                custom_execution.get("method")
+                if isinstance(custom_execution, Mapping)
+                else None
+            ),
+        },
+        extra={
+            "validation": validation_execution,
+            "drift": drift_execution,
+            "custom": custom_execution,
+        },
+    )
 
     return GateResult({
         "status": status,
@@ -400,13 +437,5 @@ def fail() -> None:
         },
         "reasons": reasons,
         "checks": checks,
-        "execution": {
-            "full_materialization": any(
-                bool(block.get("full_materialization"))
-                for block in execution_blocks
-            ),
-            "validation": validation_execution,
-            "drift": drift_execution,
-            "custom": custom_execution,
-        },
+        "execution": gate_execution,
     })

From 216adbe19e3aa5627746bc8e125c8cd410e742cb Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:37:57 +0530
Subject: [PATCH 299/439] test: lock execution provenance schema

---
 tests/test_provenance.py | 116 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 116 insertions(+)
 create mode 100644 tests/test_provenance.py

diff --git a/tests/test_provenance.py b/tests/test_provenance.py
new file mode 100644
index 0000000..fc40f9b
--- /dev/null
+++ b/tests/test_provenance.py
@@ -0,0 +1,116 @@
+from framevitals.provenance import (
+    EXECUTION_SCHEMA_VERSION,
+    execution_provenance,
+    load_fully_materializes,
+    normalize_execution,
+)
+from framevitals.sources import DatasetMetadata
+
+
+def test_execution_provenance_uses_shared_v1_contract():
+    source = {
+        "name": "data.parquet",
+        "kind": "file",
+        "format": "parquet",
+        "rows": 10_000,
+        "columns": 4,
+        "size_bytes": 1234,
+        "materialized": False,
+        "supports_projection": True,
+        "supports_streaming": True,
+    }
+
+    result = execution_provenance(
+        "bounded_example",
+        full_materialization=False,
+        source=source,
+        sampled=True,
+        source_rows=10_000,
+        source_columns=4,
+        sample_rows=1_000,
+        strategy="evenly_spaced",
+        components={"missingness": "exact", "distribution": "sample"},
+        reason="Bounded work.",
+        scope="example",
+        extra={"pair_budget": 20},
+    )
+
+    assert result["execution_schema_version"] == EXECUTION_SCHEMA_VERSION == "1"
+    assert result["method"] == "bounded_example"
+    assert result["full_materialization"] is False
+    assert result["sampled"] is True
+    assert result["source_rows"] == 10_000
+    assert result["sample_rows"] == 1_000
+    assert result["source"] == source
+    assert result["components"]["missingness"] == "exact"
+    assert result["pair_budget"] == 20
+
+
+def test_execution_provenance_omits_unavailable_optional_values():
+    result = execution_provenance(
+        "schema_only",
+        full_materialization=False,
+    )
+
+    assert result == {
+        "execution_schema_version": "1",
+        "method": "schema_only",
+        "full_materialization": False,
+    }
+
+
+def test_normalize_execution_preserves_legacy_fields_and_adds_contract():
+    result = normalize_execution({
+        "scope": "bounded_deep_statistics",
+        "sampled": True,
+        "sample_rows": 500,
+        "legacy_flag": "kept",
+    })
+
+    assert result["execution_schema_version"] == "1"
+    assert result["method"] == "bounded_deep_statistics"
+    assert result["scope"] == "bounded_deep_statistics"
+    assert result["sampled"] is True
+    assert result["sample_rows"] == 500
+    assert result["legacy_flag"] == "kept"
+    assert result["full_materialization"] is False
+
+
+def test_materialization_semantics_are_not_file_specific():
+    pandas_metadata = DatasetMetadata(
+        name="<dataframe>",
+        kind="memory",
+        format="pandas",
+        rows=3,
+        columns=1,
+        size_bytes=100,
+        materialized=True,
+        supports_projection=True,
+        supports_streaming=False,
+    )
+    arrow_metadata = DatasetMetadata(
+        name="<arrow_table>",
+        kind="memory",
+        format="arrow",
+        rows=3,
+        columns=1,
+        size_bytes=24,
+        materialized=True,
+        supports_projection=True,
+        supports_streaming=True,
+    )
+    relation_metadata = DatasetMetadata(
+        name="<duckdb_relation>",
+        kind="relation",
+        format="duckdb",
+        rows=3,
+        columns=1,
+        size_bytes=None,
+        materialized=False,
+        supports_projection=True,
+        supports_streaming=True,
+    )
+
+    assert load_fully_materializes(pandas_metadata) is False
+    assert load_fully_materializes(arrow_metadata) is True
+    assert load_fully_materializes(relation_metadata) is True

From 623a88449a4c58ea46b13582650da7085df4f56a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:38:20 +0530
Subject: [PATCH 300/439] refactor: version budgeted execution metadata

---
 src/framevitals/budgeted_analysis.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/framevitals/budgeted_analysis.py b/src/framevitals/budgeted_analysis.py
index 9ba5491..a5b1638 100644
--- a/src/framevitals/budgeted_analysis.py
+++ b/src/framevitals/budgeted_analysis.py
@@ -15,6 +15,7 @@
 from framevitals.anomaly_ensemble import detect_anomalies_ensemble
 from framevitals.deep_statistics_v2 import run_deep_statistics_v2
 from framevitals.execution import ExecutionBudget, deterministic_sample_frame
+from framevitals.provenance import normalize_execution
 from framevitals.time_series import detect_and_analyze_time_series
 
 
@@ -26,11 +27,15 @@ def _attach_execution(
     scope: str,
 ) -> dict[str, Any]:
     result = dict(payload)
-    result["execution"] = {
-        "scope": scope,
-        "scale_class": budget.scale_class,
-        **sampling,
-    }
+    result["execution"] = normalize_execution(
+        {
+            "scope": scope,
+            "scale_class": budget.scale_class,
+            **sampling,
+        },
+        method=scope,
+        full_materialization=False,
+    )
     return result
 
 

From 848e3fe1d0be84a8e8101d4683061e54391a3ab1 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:40:21 +0530
Subject: [PATCH 301/439] refactor: standardize focused execution provenance

---
 src/framevitals/focused.py | 104 ++++++++++++++++++++++++++++++++-----
 1 file changed, 90 insertions(+), 14 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index c4f3ec5..eb79201 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -8,15 +8,19 @@
 
 from __future__ import annotations
 
-from pathlib import Path
 from typing import Any
 
 import pandas as pd
 
+from framevitals.provenance import (
+    execution_provenance,
+    load_fully_materializes,
+    normalize_execution,
+)
 from framevitals.sources import StreamingDatasetSource, resolve_source
 
 
-DataInput = str | Path | pd.DataFrame
+DataInput = Any
 
 
 def _load(data: DataInput, *, label: str = "Dataset") -> tuple[pd.DataFrame, str]:
@@ -198,7 +202,7 @@ def statistics(
             budget=budget,
             max_pairs=max_pairs,
         )
-        execution = payload.setdefault("execution", {})
+        execution = dict(payload.get("execution", {}))
         sampled = len(sample) < source_rows
         execution.update({
             "sampled": sampled,
@@ -218,6 +222,12 @@ def statistics(
                 else "The streaming source fits within the deep-statistics execution budget."
             ),
         })
+        payload["execution"] = normalize_execution(
+            execution,
+            method="bounded_deep_statistics",
+            full_materialization=False,
+            source=metadata.to_dict(),
+        )
         payload["source"] = metadata.to_dict()
         return _named(payload, metadata.name)
 
@@ -232,6 +242,13 @@ def statistics(
         budget=budget,
         max_pairs=max_pairs,
     )
+    payload["execution"] = normalize_execution(
+        payload.get("execution", {}),
+        method="bounded_deep_statistics",
+        full_materialization=load_fully_materializes(metadata),
+        source=metadata.to_dict(),
+    )
+    payload["source"] = metadata.to_dict()
     return _named(payload, metadata.name)
 
 
@@ -263,17 +280,22 @@ def anomalies(
         budget = derive_execution_budget(source_rows, source_columns, mode=mode)
         numeric_columns = numeric_columns_for_streaming_source(source)
         if not numeric_columns:
+            execution = execution_provenance(
+                "bounded_anomaly_detection",
+                full_materialization=False,
+                source=metadata.to_dict(),
+                sampled=False,
+                source_rows=source_rows,
+                source_columns=source_columns,
+                sample_rows=0,
+                strategy="streaming_schema_only",
+                scope="bounded_anomaly_detection",
+                extra={"projected_columns": 0},
+            )
             return _named({
                 "available": False,
                 "reason": "No numeric columns available for anomaly analysis.",
-                "execution": {
-                    "sampled": False,
-                    "source_rows": source_rows,
-                    "source_columns": source_columns,
-                    "sample_rows": 0,
-                    "full_materialization": False,
-                    "strategy": "streaming_schema_only",
-                },
+                "execution": execution,
                 "source": metadata.to_dict(),
             }, metadata.name)
 
@@ -291,7 +313,7 @@ def anomalies(
             max_columns=max_columns,
             top_k=top_k,
         )
-        execution = payload.setdefault("execution", {})
+        execution = dict(payload.get("execution", {}))
         sampled = len(sample) < source_rows
         execution.update({
             "sampled": sampled,
@@ -312,6 +334,12 @@ def anomalies(
                 else "The streaming source fits within the anomaly execution budget."
             ),
         })
+        payload["execution"] = normalize_execution(
+            execution,
+            method="bounded_anomaly_detection",
+            full_materialization=False,
+            source=metadata.to_dict(),
+        )
         payload["source"] = metadata.to_dict()
         return _named(payload, metadata.name)
 
@@ -329,6 +357,13 @@ def anomalies(
         max_columns=max_columns,
         top_k=top_k,
     )
+    payload["execution"] = normalize_execution(
+        payload.get("execution", {}),
+        method="bounded_anomaly_detection",
+        full_materialization=load_fully_materializes(metadata),
+        source=metadata.to_dict(),
+    )
+    payload["source"] = metadata.to_dict()
     return _named(payload, metadata.name)
 
 
@@ -365,17 +400,37 @@ def relationships(
             max_candidate_pairs=max_candidate_pairs,
             max_edges_returned=max_edges_returned,
         )
+        source_rows = int(metadata.rows or len(sample))
+        source_columns = int(metadata.columns or len(sample.columns))
+        sampled = source_rows > len(sample)
         sample_metadata = payload.setdefault("sample", {})
         sample_metadata.update({
-            "source_rows": int(metadata.rows or len(sample)),
+            "source_rows": source_rows,
             "sample_rows": int(len(sample)),
-            "sampled": bool((metadata.rows or len(sample)) > len(sample)),
+            "sampled": sampled,
             "full_materialization": False,
             "strategy": "streaming_evenly_spaced_global_rows",
         })
         payload["source"] = metadata.to_dict()
         payload["streaming_source"] = True
         payload["full_materialization"] = False
+        payload["execution"] = execution_provenance(
+            "bounded_relationship_graph",
+            full_materialization=False,
+            source=metadata.to_dict(),
+            sampled=sampled,
+            source_rows=source_rows,
+            source_columns=source_columns,
+            sample_rows=int(len(sample)),
+            strategy="streaming_evenly_spaced_global_rows",
+            components={
+                "numeric_projection": "schema_exact",
+                "relationship_candidates": (
+                    "bounded_row_sample" if sampled else "full_input"
+                ),
+            },
+            extra={"projected_columns": int(len(numeric_columns))},
+        )
         return _named(payload, metadata.name)
 
     dataframe = source.load()
@@ -387,6 +442,27 @@ def relationships(
         max_candidate_pairs=max_candidate_pairs,
         max_edges_returned=max_edges_returned,
     )
+    sample_metadata = payload.get("sample", {})
+    if not isinstance(sample_metadata, dict):
+        sample_metadata = {}
+    sampled = bool(sample_metadata.get("sampled", False))
+    sample_rows = int(sample_metadata.get("sample_rows", len(dataframe)))
+    strategy = str(sample_metadata.get("strategy", "full_input"))
+    payload["source"] = metadata.to_dict()
+    payload["full_materialization"] = load_fully_materializes(metadata)
+    payload["execution"] = execution_provenance(
+        "bounded_relationship_graph",
+        full_materialization=load_fully_materializes(metadata),
+        source=metadata.to_dict(),
+        sampled=sampled,
+        source_rows=int(len(dataframe)),
+        source_columns=int(len(dataframe.columns)),
+        sample_rows=sample_rows,
+        strategy=strategy,
+        components={
+            "relationship_candidates": "bounded_row_sample" if sampled else "full_input",
+        },
+    )
     return _named(payload, metadata.name)
 
 

From 48be4205eb514b8da1e58119c42f95370886beff Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:40:56 +0530
Subject: [PATCH 302/439] test: require execution schema on streaming
 statistics

---
 tests/test_statistics_streaming.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_statistics_streaming.py b/tests/test_statistics_streaming.py
index 88c1a9b..9b30513 100644
--- a/tests/test_statistics_streaming.py
+++ b/tests/test_statistics_streaming.py
@@ -36,6 +36,8 @@ def fail_load(self):
 
     assert result["dataset_name"] == "statistics.parquet"
     execution = result["execution"]
+    assert execution["execution_schema_version"] == "1"
+    assert execution["method"] == "bounded_deep_statistics"
     assert execution["scope"] == "bounded_deep_statistics"
     assert execution["full_materialization"] is False
     assert execution["source_rows"] == len(frame)
@@ -44,4 +46,5 @@ def fail_load(self):
     assert execution["sampled"] is True
     assert execution["strategy"] == "streaming_evenly_spaced_global_rows"
     assert execution["pair_budget"] == 2
+    assert execution["source"]["format"] == "parquet"
     assert result["source"]["supports_streaming"] is True

From e1af9f9faaba26300285806336ec0bc8f7c65712 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:41:08 +0530
Subject: [PATCH 303/439] test: lock public execution schema v1

---
 tests/test_public_execution_schema.py | 104 ++++++++++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100644 tests/test_public_execution_schema.py

diff --git a/tests/test_public_execution_schema.py b/tests/test_public_execution_schema.py
new file mode 100644
index 0000000..92aea9f
--- /dev/null
+++ b/tests/test_public_execution_schema.py
@@ -0,0 +1,104 @@
+import numpy as np
+import pandas as pd
+
+import framevitals as fv
+from framevitals.sources import DatasetMetadata
+
+
+def _frame(rows: int = 800) -> pd.DataFrame:
+    values = np.arange(rows, dtype=np.float64)
+    return pd.DataFrame({
+        "value": values,
+        "other": values * 2.0 + 1.0,
+        "third": np.sin(values / 10.0),
+    })
+
+
+class _LoadedSource:
+    def __init__(self, frame: pd.DataFrame):
+        self.frame = frame
+
+    def inspect(self):
+        return DatasetMetadata(
+            name="loaded-source",
+            kind="remote",
+            format="custom",
+            rows=len(self.frame),
+            columns=len(self.frame.columns),
+            size_bytes=None,
+            materialized=False,
+            supports_projection=False,
+            supports_streaming=False,
+        )
+
+    def load(self):
+        return self.frame.copy()
+
+
+def test_dataframe_statistics_use_execution_schema_v1():
+    result = fv.statistics(_frame(), mode="quick", max_pairs=2)
+    execution = result["execution"]
+
+    assert execution["execution_schema_version"] == "1"
+    assert execution["method"] == "bounded_deep_statistics"
+    assert execution["scope"] == "bounded_deep_statistics"
+    assert execution["full_materialization"] is False
+    assert execution["source"]["format"] == "pandas"
+
+
+def test_non_pandas_statistics_report_full_materialization():
+    result = fv.statistics(_LoadedSource(_frame()), mode="quick", max_pairs=2)
+    execution = result["execution"]
+
+    assert execution["execution_schema_version"] == "1"
+    assert execution["method"] == "bounded_deep_statistics"
+    assert execution["full_materialization"] is True
+    assert execution["source"]["format"] == "custom"
+
+
+def test_dataframe_anomalies_use_execution_schema_v1():
+    result = fv.anomalies(
+        _frame(),
+        mode="quick",
+        max_columns=3,
+        top_k=5,
+    )
+    execution = result["execution"]
+
+    assert execution["execution_schema_version"] == "1"
+    assert execution["method"] == "bounded_anomaly_detection"
+    assert execution["scope"] == "bounded_anomaly_detection"
+    assert execution["full_materialization"] is False
+    assert execution["source"]["format"] == "pandas"
+
+
+def test_relationships_use_execution_schema_v1_and_preserve_legacy_sample():
+    result = fv.relationships(_frame(), max_sample_rows=64)
+    execution = result["execution"]
+
+    assert execution["execution_schema_version"] == "1"
+    assert execution["method"] == "bounded_relationship_graph"
+    assert execution["full_materialization"] is False
+    assert execution["sampled"] is True
+    assert execution["sample_rows"] == 64
+    assert execution["source_rows"] == 800
+    assert execution["source"]["format"] == "pandas"
+
+    assert result["sample"]["sampled"] is True
+    assert result["sample"]["sample_rows"] == 64
+    assert result["full_materialization"] is False
+
+
+def test_exact_validation_and_gate_use_execution_schema_v1():
+    frame = _frame(100)
+    contract = fv.infer_contract(frame)
+
+    validation = fv.validate(frame, contract)
+    assert validation["execution"]["execution_schema_version"] == "1"
+    assert validation["execution"]["method"] == "exact_contract_validation"
+    assert validation["execution"]["sampled"] is False
+
+    gate = fv.gate(frame, contract=contract)
+    assert gate["execution"]["execution_schema_version"] == "1"
+    assert gate["execution"]["method"] == "quality_gate"
+    assert gate["execution"]["validation"]["execution_schema_version"] == "1"

From 9a56a44dc16119b2a7fba98ef446d74c1c41360d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:42:31 +0530
Subject: [PATCH 304/439] refactor: version streaming role provenance

---
 src/framevitals/streaming_roles.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/framevitals/streaming_roles.py b/src/framevitals/streaming_roles.py
index 8f8635d..5362613 100644
--- a/src/framevitals/streaming_roles.py
+++ b/src/framevitals/streaming_roles.py
@@ -20,6 +20,7 @@
     infer_column_roles,
     summarize_roles,
 )
+from framevitals.provenance import normalize_execution
 
 
 _CARDINALITY_ROLES = {
@@ -208,10 +209,8 @@ def infer_streaming_column_roles(
         })
         columns[str(column)] = info
 
-    return {
-        "columns": columns,
-        "summary": summarize_roles(columns),
-        "execution": {
+    execution = normalize_execution(
+        {
             "method": "streaming_profile_with_bounded_semantic_sample",
             "full_materialization": False,
             "source_rows": source_rows,
@@ -224,4 +223,11 @@ def infer_streaming_column_roles(
                 "categorical_cardinality_without_native_sketch",
             ],
         },
+        method="streaming_profile_with_bounded_semantic_sample",
+        full_materialization=False,
+    )
+    return {
+        "columns": columns,
+        "summary": summarize_roles(columns),
+        "execution": execution,
     }

From 9957e21ef70e6c2459bf6e6fc5e4c9e7db4fd6ee Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:42:52 +0530
Subject: [PATCH 305/439] refactor: version streaming health provenance

---
 src/framevitals/health_score.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/framevitals/health_score.py b/src/framevitals/health_score.py
index aee9d7d..0e0f87e 100644
--- a/src/framevitals/health_score.py
+++ b/src/framevitals/health_score.py
@@ -5,6 +5,8 @@
 import numpy as np
 import pandas as pd
 
+from framevitals.provenance import normalize_execution
+
 
 def calculate_outlier_percent(df: pd.DataFrame):
     numeric_cols = df.select_dtypes(include=[np.number]).columns
@@ -173,18 +175,13 @@ def calculate_health_score_from_profile_sample(
         isinstance(duplicate_metadata, Mapping) and duplicate_metadata.get("sampled")
     )
 
-    return _score_payload(
-        missing_percent=missing_percent,
-        duplicate_percent=duplicate_percent,
-        outlier_percent=outlier_percent,
-        constant_columns=constant_columns,
-        high_cardinality_columns=high_cardinality_columns,
-        columns=columns,
-        outlier_details=outlier_details,
-        execution={
+    execution = normalize_execution(
+        {
             "method": "streaming_profile_with_bounded_row_sample",
             "source_rows": rows,
+            "source_columns": columns,
             "sample_rows": sample_rows,
+            "sampled": sample_rows < rows,
             "full_materialization": False,
             "components": {
                 "completeness": "full_stream_exact",
@@ -197,6 +194,18 @@ def calculate_health_score_from_profile_sample(
                 ),
             },
         },
+        method="streaming_profile_with_bounded_row_sample",
+        full_materialization=False,
+    )
+    return _score_payload(
+        missing_percent=missing_percent,
+        duplicate_percent=duplicate_percent,
+        outlier_percent=outlier_percent,
+        constant_columns=constant_columns,
+        high_cardinality_columns=high_cardinality_columns,
+        columns=columns,
+        outlier_details=outlier_details,
+        execution=execution,
     )
 
 

From 4f667fbca6fd08eccdbd7390c22010fe3e76f152 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:43:09 +0530
Subject: [PATCH 306/439] refactor: version streaming ML-readiness provenance

---
 src/framevitals/ml_readiness.py | 33 ++++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/src/framevitals/ml_readiness.py b/src/framevitals/ml_readiness.py
index 1d18c62..2d97403 100644
--- a/src/framevitals/ml_readiness.py
+++ b/src/framevitals/ml_readiness.py
@@ -8,6 +8,8 @@
 
 import numpy as np
 
+from framevitals.provenance import normalize_execution
+
 
 _CATEGORICAL_DTYPES = ["object", "string", "category", "bool"]
 
@@ -85,20 +87,25 @@ def calculate_ml_readiness_from_profile(profile: dict[str, Any]) -> dict[str, An
     if streaming.get("enabled"):
         duplicate_metadata = profile.get("duplicate_metadata", {})
         duplicate_sampled = bool(duplicate_metadata.get("sampled"))
-        result["execution"] = {
-            "method": "streaming_profile",
-            "full_materialization": bool(streaming.get("full_materialization", False)),
-            "source_rows": rows,
-            "source_columns": columns,
-            "sample_rows": int(streaming.get("sample_rows", 0) or 0),
-            "components": {
-                "missingness": "full_stream_exact",
-                "column_groups": "schema_exact",
-                "duplicate_rate": (
-                    "bounded_row_sample_estimate" if duplicate_sampled else "exact"
-                ),
+        result["execution"] = normalize_execution(
+            {
+                "method": "streaming_profile",
+                "full_materialization": bool(streaming.get("full_materialization", False)),
+                "source_rows": rows,
+                "source_columns": columns,
+                "sample_rows": int(streaming.get("sample_rows", 0) or 0),
+                "sampled": int(streaming.get("sample_rows", 0) or 0) < rows,
+                "components": {
+                    "missingness": "full_stream_exact",
+                    "column_groups": "schema_exact",
+                    "duplicate_rate": (
+                        "bounded_row_sample_estimate" if duplicate_sampled else "exact"
+                    ),
+                },
             },
-        }
+            method="streaming_profile",
+            full_materialization=bool(streaming.get("full_materialization", False)),
+        )
 
     return result
 

From d9f65e56a549783c3af3cc525d88ebb893aa3d2c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:43:24 +0530
Subject: [PATCH 307/439] refactor: version streaming quality provenance

---
 src/framevitals/streaming_quality.py | 57 +++++++++++++++-------------
 1 file changed, 31 insertions(+), 26 deletions(-)

diff --git a/src/framevitals/streaming_quality.py b/src/framevitals/streaming_quality.py
index 662cea0..77fc626 100644
--- a/src/framevitals/streaming_quality.py
+++ b/src/framevitals/streaming_quality.py
@@ -14,6 +14,7 @@
 import pandas as pd
 
 from framevitals.column_roles import infer_column_roles
+from framevitals.provenance import normalize_execution
 from framevitals.quality_diagnostics import run_quality_diagnostics
 
 
@@ -147,30 +148,34 @@ def run_streaming_quality_diagnostics(
         "issue_count": issue_count + (1 if duplicate_rows else 0),
         "primary_key_candidate_count": len(payload.get("primary_key_candidates", [])),
     }
-    payload["execution"] = {
-        "method": "streaming_profile_with_bounded_quality_sample",
-        "full_materialization": False,
-        "source_rows": int(source_rows),
-        "source_columns": int(source_columns),
-        "sample_rows": sample_rows,
-        "sampled": sample_rows < int(source_rows),
-        "full_source_inputs": ["missingness", "duplicate_row_estimate"],
-        "sample_inputs": [
-            "column_roles",
-            "identifier_duplicates",
-            "quasi_constants",
-            "duplicate_column_candidates",
-            "coercion_candidates",
-            "category_normalisation",
-            "blank_strings",
-            "infinite_values",
-            "mixed_object_types",
-            "missingness_relationships",
-        ],
-        "candidate_only_checks": (
-            ["primary_key_candidates", "duplicate_columns"]
-            if sample_rows < int(source_rows)
-            else []
-        ),
-    }
+    payload["execution"] = normalize_execution(
+        {
+            "method": "streaming_profile_with_bounded_quality_sample",
+            "full_materialization": False,
+            "source_rows": int(source_rows),
+            "source_columns": int(source_columns),
+            "sample_rows": sample_rows,
+            "sampled": sample_rows < int(source_rows),
+            "full_source_inputs": ["missingness", "duplicate_row_estimate"],
+            "sample_inputs": [
+                "column_roles",
+                "identifier_duplicates",
+                "quasi_constants",
+                "duplicate_column_candidates",
+                "coercion_candidates",
+                "category_normalisation",
+                "blank_strings",
+                "infinite_values",
+                "mixed_object_types",
+                "missingness_relationships",
+            ],
+            "candidate_only_checks": (
+                ["primary_key_candidates", "duplicate_columns"]
+                if sample_rows < int(source_rows)
+                else []
+            ),
+        },
+        method="streaming_profile_with_bounded_quality_sample",
+        full_materialization=False,
+    )
     return payload

From 0202cb207830bb755083f6efbe349a56ac0cbdfc Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:44:01 +0530
Subject: [PATCH 308/439] docs: add FrameVitals documentation home

---
 docs/index.md | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 docs/index.md

diff --git a/docs/index.md b/docs/index.md
new file mode 100644
index 0000000..babd9fa
--- /dev/null
+++ b/docs/index.md
@@ -0,0 +1,99 @@
+# FrameVitals
+
+**FrameVitals is a source-aware data-health and quality-gate engine for tabular pipelines.**
+
+It provides one workflow for inspecting data health, comparing production batches,
+validating contracts, enforcing domain-specific invariants, and retaining compact
+monitoring history.
+
+```python
+import framevitals as fv
+
+source = fv.inspect_source("production.parquet")
+report = fv.analyze("production.parquet", mode="quick")
+drift = fv.compare("training.parquet", "production.parquet")
+validation = fv.validate("production.parquet", contract)
+gate = fv.gate(
+    "production.parquet",
+    reference="training.parquet",
+    contract=contract,
+)
+```
+
+## Core design
+
+FrameVitals is built around four constraints:
+
+1. **Source-aware execution** — supported sources expose metadata, projection, and
+   batches before FrameVitals decides whether pandas materialization is necessary.
+2. **Bounded expensive work** — statistics, anomaly detection, relationship discovery,
+   and drift use explicit execution budgets when full-row computation is not required.
+3. **Exactness where correctness requires it** — contracts and arbitrary custom Python
+   invariants are not silently weakened to sampled checks.
+4. **Execution transparency** — public results disclose whether work was exact,
+   sampled, estimated, streamed, or fully materialized.
+
+## Main workflow
+
+```text
+source
+  │
+  ├─► inspect_source
+  │
+  ├─► analyze ─► snapshot ─► SnapshotHistory
+  │
+  ├─► compare(reference, current)
+  │
+  ├─► validate(current, contract)
+  │
+  └─► gate
+        ├─ contract validation
+        ├─ drift
+        └─ custom checks / plugins
+```
+
+## Supported source families
+
+FrameVitals currently recognizes:
+
+- pandas `DataFrame` inputs;
+- CSV and TSV files, with optional Arrow streaming;
+- Parquet through the Arrow capability;
+- PyArrow `Table` and `RecordBatch` inputs;
+- table producers supporting the Arrow C Stream / PyCapsule interface;
+- lazy DuckDB relations through the optional DuckDB adapter;
+- custom objects implementing the FrameVitals `DatasetSource` protocol.
+
+Use `fv.inspect_source(data)` before analysis when you want to see the source's
+shape metadata and streaming/projection capabilities.
+
+## Installation
+
+```bash
+pip install framevitals
+```
+
+Optional capabilities are deliberately separated:
+
+```bash
+pip install "framevitals[arrow]"
+pip install "framevitals[duckdb]"
+pip install "framevitals[excel]"
+pip install "framevitals[plot]"
+pip install "framevitals[ml]"
+pip install "framevitals[ai]"
+pip install "framevitals[web]"
+```
+
+For documentation development:
+
+```bash
+pip install -e ".[docs]"
+mkdocs serve
+```
+
+## Documentation map
+
+- [Source-aware execution](source-execution.md)
+- [Execution provenance](execution-provenance.md)
+- [Quality gates and custom checks](quality-gates.md)

From 6ca09bb8b35bbb9aabcaf621a46d63ab392058be Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:44:15 +0530
Subject: [PATCH 309/439] docs: define execution provenance contract

---
 docs/execution-provenance.md | 119 +++++++++++++++++++++++++++++++++++
 1 file changed, 119 insertions(+)
 create mode 100644 docs/execution-provenance.md

diff --git a/docs/execution-provenance.md b/docs/execution-provenance.md
new file mode 100644
index 0000000..609318e
--- /dev/null
+++ b/docs/execution-provenance.md
@@ -0,0 +1,119 @@
+# Execution provenance
+
+FrameVitals results can mix exact metadata, full-stream calculations, bounded row
+samples, estimates, and operations that intentionally materialize a complete pandas
+DataFrame. The `execution` block makes those decisions machine-readable.
+
+## Schema version
+
+The shared execution contract currently uses:
+
+```json
+{
+  "execution_schema_version": "1"
+}
+```
+
+During the `0.x` series, older operation-specific keys remain available while public
+results converge on this common vocabulary.
+
+## Common fields
+
+| Field | Meaning |
+| --- | --- |
+| `execution_schema_version` | Version of the shared execution metadata contract |
+| `method` | Stable description of the execution strategy |
+| `full_materialization` | Whether FrameVitals created a complete pandas representation of a non-pandas source |
+| `source` | Source metadata/capabilities when available |
+| `sampled` | Whether row-level work used fewer than all source rows |
+| `source_rows` | Exact source row count when known |
+| `source_columns` | Exact source column count when known |
+| `sample_rows` | Number of rows used for bounded row-level work |
+| `strategy` | Sampling or source-consumption strategy |
+| `components` | Per-component exactness/approximation information |
+| `reason` | Human-readable explanation for the execution choice |
+| `scope` | Legacy/operation-specific scope retained during the `0.x` migration |
+
+Not every field appears in every result. Missing information is omitted rather than
+represented as ambiguous placeholder values.
+
+## Full materialization semantics
+
+`full_materialization` describes **what the operation did**, not merely where the
+source came from.
+
+A pandas `DataFrame` is already materialized when supplied by the caller, so running
+an exact operation on it does not count as FrameVitals materializing a new source.
+By contrast, converting any of these into a complete pandas DataFrame does count:
+
+- a file-backed source;
+- a PyArrow table;
+- a DuckDB relation;
+- a remote/custom source.
+
+This distinction matters for memory planning and for pipelines that deliberately stay
+on Arrow/relation-backed execution paths.
+
+## Examples
+
+### Bounded statistics
+
+```json
+{
+  "execution_schema_version": "1",
+  "method": "bounded_deep_statistics",
+  "full_materialization": false,
+  "sampled": true,
+  "source_rows": 120000,
+  "source_columns": 18,
+  "sample_rows": 1000,
+  "strategy": "streaming_evenly_spaced_global_rows"
+}
+```
+
+### Exact contract validation
+
+```json
+{
+  "execution_schema_version": "1",
+  "method": "exact_contract_validation",
+  "full_materialization": true,
+  "sampled": false,
+  "source": {
+    "kind": "relation",
+    "format": "duckdb"
+  }
+}
+```
+
+### Quality gate
+
+A gate aggregates the execution blocks of the selected check families:
+
+```python
+result = fv.gate(
+    current,
+    reference=reference,
+    contract=contract,
+    custom_checks=[positive_revenue],
+)
+
+print(result["execution"])
+```
+
+The top-level gate reports `full_materialization=True` when **any** selected check
+family required it, while nested `validation`, `drift`, and `custom` entries preserve
+family-level provenance.
+
+## Exactness is diagnostic-specific
+
+A streaming-capable source does not make every metric approximate. For example:
+
+- source shape and schema can be exact;
+- missingness can be computed over the full stream;
+- duplicate rate may become a bounded estimate on a large source;
+- deep statistics may use a deterministic bounded sample;
+- contract uniqueness checks stay exact and therefore may materialize.
+
+Consumers should prefer the `execution`/`components` fields over assumptions based on
+file format or dataset size.

From c4bfee2447ba5580c7883d9f52446398c5653df7 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:44:31 +0530
Subject: [PATCH 310/439] docs: explain source-aware execution

---
 docs/source-execution.md | 154 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 154 insertions(+)
 create mode 100644 docs/source-execution.md

diff --git a/docs/source-execution.md b/docs/source-execution.md
new file mode 100644
index 0000000..ec11600
--- /dev/null
+++ b/docs/source-execution.md
@@ -0,0 +1,154 @@
+# Source-aware execution
+
+FrameVitals separates **what the data is stored in** from **what a diagnostic needs to
+compute**. Public APIs resolve inputs into a `DatasetSource` before choosing between
+batch streaming and pandas materialization.
+
+## Inspect a source first
+
+```python
+import framevitals as fv
+
+info = fv.inspect_source(data)
+print(info)
+```
+
+Typical fields include:
+
+```json
+{
+  "name": "production.parquet",
+  "kind": "file",
+  "format": "parquet",
+  "rows": 1200000,
+  "columns": 42,
+  "size_bytes": 84599210,
+  "materialized": false,
+  "supports_projection": true,
+  "supports_streaming": true
+}
+```
+
+`inspect_source()` does not run health/statistical diagnostics. Some adapters may need
+to scan lightweight metadata (for example exact relation row counts), but they do not
+materialize the full dataset into pandas simply to describe source capabilities.
+
+## DatasetSource protocol
+
+A minimal source provides:
+
+```python
+class DatasetSource(Protocol):
+    def inspect(self) -> DatasetMetadata: ...
+    def load(self) -> pandas.DataFrame: ...
+```
+
+A streaming source additionally provides:
+
+```python
+class StreamingDatasetSource(DatasetSource, Protocol):
+    def iter_batches(
+        self,
+        *,
+        batch_size: int = 65_536,
+        columns: Sequence[str] | None = None,
+    ) -> Iterator[Any]: ...
+```
+
+This means new storage systems can integrate with FrameVitals without changing every
+diagnostic implementation.
+
+## Current adapters
+
+### pandas
+
+A pandas `DataFrame` is already materialized. It remains the reference in-memory
+representation for exact/full-row operations.
+
+### Parquet
+
+With `framevitals[arrow]`, Parquet supports:
+
+- exact row/column metadata from file metadata;
+- column projection;
+- Arrow record batches;
+- bounded global row sampling without `ParquetSource.load()`.
+
+### CSV / TSV
+
+With Arrow installed, compatible delimited files use incremental Arrow CSV reading.
+If Arrow cannot interpret a file, FrameVitals preserves compatibility by falling back
+to the existing pandas loader rather than silently returning a different parse.
+
+### PyArrow Table and RecordBatch
+
+PyArrow in-memory objects enter the batch path directly. FrameVitals does not first
+convert the complete object to pandas for streaming-safe diagnostics.
+
+### Arrow C Stream / PyCapsule producers
+
+Table-like objects implementing `__arrow_c_stream__` can be normalized through
+PyArrow. This gives interoperable support to producers such as modern dataframe
+libraries without hard-coding each library into FrameVitals.
+
+A raw `RecordBatchReader` is currently rejected because it does not expose the cheap
+exact row count required by FrameVitals' current planning/profiling contract.
+
+### DuckDB relations
+
+With `framevitals[duckdb]`, a lazy `DuckDBPyRelation` supports:
+
+- exact row count via a cached aggregate;
+- schema inspection;
+- projection pushed into the DuckDB relation;
+- Arrow `RecordBatchReader` transport;
+- pandas materialization only when an exact/full-row API explicitly requires it.
+
+```python
+import duckdb
+import framevitals as fv
+
+con = duckdb.connect()
+relation = con.sql("SELECT * FROM read_parquet('events/*.parquet')")
+
+print(fv.inspect_source(relation))
+report = fv.analyze(relation, mode="quick")
+```
+
+## Source-aware public APIs
+
+The source abstraction is shared by:
+
+- `inspect_source()`
+- `profile()`
+- `roles()`
+- `health()`
+- `ml_readiness()`
+- `quality()`
+- `statistics()`
+- `anomalies()`
+- `relationships()`
+- `target_analysis()`
+- `analyze()`
+- `plan()`
+- `compare()`
+- `validate()`
+- `run_checks()`
+- `gate()`
+
+The operation still decides whether streaming is semantically valid. For example,
+contract validation and arbitrary Python checks deliberately remain exact.
+
+## Adding a source adapter
+
+Contributors should preserve these invariants:
+
+1. `inspect()` should be cheaper/safer than full pandas materialization.
+2. `iter_batches()` must respect requested projection where supported.
+3. source metadata must report the **true** source shape, not sample shape.
+4. bounded algorithms must disclose sampling strategy and row counts.
+5. exact operations must not claim streaming merely because the source supports it.
+6. optional storage dependencies must remain lazily imported.
+
+See [Execution provenance](execution-provenance.md) for the metadata contract used to
+report these choices.

From 540e5a95f369effd117bfeeec6e41cb62f134089 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:44:46 +0530
Subject: [PATCH 311/439] docs: explain quality gates and extensions

---
 docs/quality-gates.md | 164 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 164 insertions(+)
 create mode 100644 docs/quality-gates.md

diff --git a/docs/quality-gates.md b/docs/quality-gates.md
new file mode 100644
index 0000000..7a778dd
--- /dev/null
+++ b/docs/quality-gates.md
@@ -0,0 +1,164 @@
+# Quality gates and custom checks
+
+FrameVitals quality gates combine the checks a pipeline actually needs into one
+`pass` / `warn` / `fail` verdict.
+
+## Built-in gate families
+
+A gate can run any combination of:
+
+- exact contract validation;
+- reference-vs-current drift;
+- exact user-defined Python checks.
+
+```python
+result = fv.gate(
+    current,
+    reference=reference,
+    contract=contract,
+    custom_checks=[positive_revenue],
+)
+
+print(result.status)
+print(result.passed)
+print(result.reasons)
+```
+
+At least one family must be selected.
+
+## Contracts
+
+Infer a reusable contract from a trusted reference dataset:
+
+```python
+contract = fv.infer_contract(reference)
+validation = fv.validate(current, contract)
+```
+
+Current contracts can encode:
+
+- required and optional columns;
+- broad dtype expectations;
+- nullability and tolerated null rates;
+- numeric bounds with configurable tolerance;
+- low-cardinality allowed values;
+- uniqueness hints.
+
+Contract validation remains exact. If the input is Arrow-, relation-, or file-backed,
+FrameVitals may materialize the complete dataset to pandas rather than weakening the
+contract to a sample.
+
+## Custom Python invariants
+
+```python
+@fv.check(
+    "positive revenue",
+    severity="error",
+    description="Revenue cannot be negative.",
+)
+def positive_revenue(df):
+    minimum = float(df["revenue"].min())
+    return {
+        "passed": minimum >= 0,
+        "message": "Negative revenue found." if minimum < 0 else "Revenue is valid.",
+        "details": {"minimum": minimum},
+    }
+```
+
+Run checks independently:
+
+```python
+result = fv.run_checks(current, [positive_revenue])
+print(result.status)
+print(result.findings)
+```
+
+Or include them in a gate:
+
+```python
+result = fv.gate(current, custom_checks=[positive_revenue])
+```
+
+Every custom check receives an isolated DataFrame copy so one check cannot mutate the
+input seen by later checks. Exceptions raised by provider code become structured error
+results instead of aborting the entire check collection.
+
+### Severity
+
+Use `severity="warning"` for soft expectations and `severity="error"` for hard
+invariants. A warning-only custom-check result has status `warn` but remains passing;
+an error failure produces status `fail`.
+
+## Third-party check plugins
+
+External packages can expose reusable checks with normal Python package entry points:
+
+```toml
+[project.entry-points."framevitals.checks"]
+positive_revenue = "acme_data_checks:positive_revenue"
+```
+
+FrameVitals **does not automatically import installed plugins**. Applications opt in:
+
+```python
+checks = fv.discover_checks()
+result = fv.gate(current, custom_checks=checks)
+```
+
+This is an intentional trust boundary: loading an entry point executes code supplied
+by the provider package.
+
+Discovery rejects duplicate public check names and surfaces broken providers rather
+than silently changing gate behavior.
+
+## Drift thresholds
+
+```python
+result = fv.gate(
+    current,
+    reference=reference,
+    drift_warn_on="moderate",
+    drift_fail_on="severe",
+)
+```
+
+Supported severities are `stable`, `minor`, `moderate`, and `severe`.
+
+## CLI
+
+```bash
+framevitals gate production.parquet \
+  --reference training.parquet \
+  --contract contract.json \
+  --drift-warn-on moderate \
+  --drift-fail-on severe \
+  --output framevitals-gate.json
+```
+
+The CLI exits `0` for pass/warn and `1` for fail.
+
+## GitHub Action
+
+The repository includes a reusable composite action:
+
+```yaml
+- uses: parthdongre/FrameVitals@v0.1.0
+  id: framevitals
+  with:
+    current: production.parquet
+    reference: training.parquet
+    contract: contract.json
+    output: framevitals-gate.json
+```
+
+For real CI pipelines, pin a released tag or commit SHA instead of a moving branch.
+The action exposes `status`, `passed`, and `result-path` outputs.
+
+## Execution transparency
+
+The gate's top-level `execution.full_materialization` becomes true if any selected
+family required full materialization. Family-specific execution blocks remain nested
+under `validation`, `drift`, and `custom` so automation can distinguish why a gate
+materialized or sampled data.
+
+See [Execution provenance](execution-provenance.md) for the shared schema.

From b9e8dcfc7d7ec76170f2c7dd070ef7ea421aba89 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:45:11 +0530
Subject: [PATCH 312/439] build: add documentation capability

---
 pyproject.toml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 8479677..438b659 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,6 +31,7 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
     "Topic :: Scientific/Engineering :: Information Analysis",
+    "Typing :: Typed",
 ]
 dependencies = [
     "pandas>=2.2",
@@ -71,6 +72,10 @@ duckdb = [
     "duckdb>=1.4",
     "pyarrow>=25.0,<26",
 ]
+docs = [
+    "mkdocs",
+    "mkdocs-material",
+]
 all = [
     "openpyxl>=3.1",
     "xlrd>=2.0",
@@ -106,7 +111,7 @@ dev = [
 framevitals = "framevitals.cli:main"
 
 [project.urls]
-Documentation = "https://github.com/parthdongre/FrameVitals#readme"
+Documentation = "https://github.com/parthdongre/FrameVitals/tree/main/docs"
 Repository = "https://github.com/parthdongre/FrameVitals"
 Issues = "https://github.com/parthdongre/FrameVitals/issues"
 Changelog = "https://github.com/parthdongre/FrameVitals/blob/main/CHANGELOG.md"

From a45a46082ca4978770269f3972c138ad48c96929 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:45:20 +0530
Subject: [PATCH 313/439] docs: configure MkDocs site

---
 mkdocs.yml | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 mkdocs.yml

diff --git a/mkdocs.yml b/mkdocs.yml
new file mode 100644
index 0000000..d2b7709
--- /dev/null
+++ b/mkdocs.yml
@@ -0,0 +1,35 @@
+site_name: FrameVitals
+site_description: Source-aware data health, drift, contracts, and quality gates for tabular pipelines.
+repo_name: parthdongre/FrameVitals
+repo_url: https://github.com/parthdongre/FrameVitals
+edit_uri: edit/dev/docs/
+
+strict: true
+
+theme:
+  name: material
+  features:
+    - navigation.sections
+    - navigation.top
+    - content.code.copy
+    - search.suggest
+
+nav:
+  - Home: index.md
+  - Source-aware execution: source-execution.md
+  - Execution provenance: execution-provenance.md
+  - Quality gates and custom checks: quality-gates.md
+
+markdown_extensions:
+  - admonition
+  - attr_list
+  - tables
+  - toc:
+      permalink: true
+  - pymdownx.highlight:
+      anchor_linenums: true
+  - pymdownx.inlinehilite
+  - pymdownx.superfences
+
+plugins:
+  - search

From 6b843d812e158bf0cba9fe3da2c065b2ea8045ec Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:45:28 +0530
Subject: [PATCH 314/439] ci: validate documentation site

---
 .github/workflows/docs.yml | 52 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 .github/workflows/docs.yml

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
new file mode 100644
index 0000000..46bf9ae
--- /dev/null
+++ b/.github/workflows/docs.yml
@@ -0,0 +1,52 @@
+name: Docs
+
+on:
+  push:
+    branches: [main, dev, "develop/august"]
+    paths:
+      - "docs/**"
+      - "mkdocs.yml"
+      - "pyproject.toml"
+      - ".github/workflows/docs.yml"
+  pull_request:
+    paths:
+      - "docs/**"
+      - "mkdocs.yml"
+      - "pyproject.toml"
+      - ".github/workflows/docs.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: strict MkDocs build
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install documentation dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[docs]"
+
+      - name: Verify dependency consistency
+        run: python -m pip check
+
+      - name: Build documentation strictly
+        run: mkdocs build --strict

From dbe886d46f4503636bc6d2a0330e53099d920ed8 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:47:49 +0530
Subject: [PATCH 315/439] feat: add source inspection CLI command

---
 src/framevitals/cli.py | 46 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/src/framevitals/cli.py b/src/framevitals/cli.py
index b4d1cbb..682e142 100644
--- a/src/framevitals/cli.py
+++ b/src/framevitals/cli.py
@@ -121,6 +121,19 @@ def build_parser() -> argparse.ArgumentParser:
     )
     _add_output_argument(analyze_parser)
 
+    inspect_parser = subparsers.add_parser(
+        "inspect",
+        help="Inspect dataset source metadata and execution capabilities.",
+    )
+    inspect_parser.add_argument("file", type=Path, help="Path to the dataset.")
+    inspect_parser.add_argument(
+        "--format",
+        choices=["terminal", "json"],
+        default="terminal",
+        help="Source metadata output format.",
+    )
+    _add_output_argument(inspect_parser)
+
     plan_parser = subparsers.add_parser(
         "plan",
         help="Preview applicable analyses without running heavy stages.",
@@ -340,6 +353,25 @@ def _emit(payload: dict, output: Path | None) -> None:
     print(json.dumps(payload, indent=2, default=str))
 
 
+def _render_source(source: dict) -> str:
+    rows = source.get("rows")
+    columns = source.get("columns")
+    size_bytes = source.get("size_bytes")
+    lines = [
+        "FrameVitals source",
+        f"Name            {source.get('name', 'unknown')}",
+        f"Kind            {source.get('kind', 'unknown')}",
+        f"Format          {source.get('format', 'unknown')}",
+        f"Rows            {rows if rows is not None else 'unknown'}",
+        f"Columns         {columns if columns is not None else 'unknown'}",
+        f"Size bytes      {size_bytes if size_bytes is not None else 'unknown'}",
+        f"Materialized    {'yes' if source.get('materialized') else 'no'}",
+        f"Projection      {'yes' if source.get('supports_projection') else 'no'}",
+        f"Streaming       {'yes' if source.get('supports_streaming') else 'no'}",
+    ]
+    return "\n".join(lines)
+
+
 def _render_validation(report: dict) -> str:
     summary = report.get("summary", {})
     lines = [
@@ -453,6 +485,20 @@ def main() -> int:
                 print(f"HTML report   {args.html_report}")
         return 0
 
+    if args.command == "inspect":
+        from framevitals.sources import inspect_source
+
+        result = inspect_source(args.file)
+        if args.output is not None:
+            _write_json(result, args.output)
+        if args.format == "json":
+            print(json.dumps(result, indent=2, default=str))
+        else:
+            print(_render_source(result))
+            if args.output is not None:
+                print(f"Source JSON    {args.output}")
+        return 0
+
     if args.command == "plan":
         from framevitals.planning_api import plan
 

From 7c682514924800c680b565997e80446a16f64bbf Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:48:15 +0530
Subject: [PATCH 316/439] test: cover source inspection CLI

---
 tests/test_cli.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/tests/test_cli.py b/tests/test_cli.py
index 2182b3c..d7b0afb 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -24,6 +24,63 @@ def test_cli_parser():
     assert args.output is None
 
 
+def test_cli_inspect_parser():
+    parser = build_parser()
+
+    args = parser.parse_args([
+        "inspect",
+        "dataset.parquet",
+        "--format",
+        "json",
+        "--output",
+        "source.json",
+    ])
+
+    assert args.command == "inspect"
+    assert args.file.name == "dataset.parquet"
+    assert args.format == "json"
+    assert args.output.name == "source.json"
+
+
+def test_cli_inspect_emits_source_metadata_and_json_file(
+    tmp_path,
+    monkeypatch,
+    capsys,
+):
+    dataset = tmp_path / "dataset.csv"
+    output = tmp_path / "source.json"
+    dataset.write_text("value,label\n1,a\n2,b\n3,c\n", encoding="utf-8")
+
+    monkeypatch.setattr(
+        "framevitals.sources.DelimitedTextSource._pyarrow_csv",
+        lambda self: None,
+    )
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "inspect",
+            str(dataset),
+            "--format",
+            "json",
+            "--output",
+            str(output),
+        ],
+    )
+
+    assert main() == 0
+    rendered = json.loads(capsys.readouterr().out)
+    saved = json.loads(output.read_text(encoding="utf-8"))
+
+    assert rendered == saved
+    assert rendered["name"] == "dataset.csv"
+    assert rendered["kind"] == "file"
+    assert rendered["format"] == "csv"
+    assert rendered["supports_streaming"] is False
+    assert rendered["supports_projection"] is False
+    assert rendered["size_bytes"] == dataset.stat().st_size
+
+
 def test_cli_target_argument():
     parser = build_parser()
 

From 2a24a980c795edb93fbe963b5be4dd107c424e5a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:50:27 +0530
Subject: [PATCH 317/439] docs: record source inspection and provenance schema

---
 CHANGELOG.md | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b26f135..c51612c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,8 @@ Development continues on the development branches. Changes intended for the next
 - Reusable root `action.yml` GitHub Action for running FrameVitals quality gates with status/result outputs
 - Path-scoped smoke CI for the reusable Gate Action
 - `framevitals plan` for previewing applicable modules, execution budgets, source capabilities, and bounded planning samples
+- `framevitals.inspect_source()` for inspecting source shape, storage kind, format, materialization state, projection support, and streaming capability without running diagnostics
+- `framevitals inspect` CLI command with terminal/JSON output and optional persisted source metadata
 - Compact versioned analysis snapshots with deterministic fingerprints, JSON persistence, and snapshot-to-snapshot schema/health/ML-readiness/finding diffs
 - `SnapshotHistory` for persistent compact monitoring timelines, latest/previous lookup, labels, and recent-state comparison without storing raw datasets
 - Dict-compatible result objects including `AnalysisResult`, `DriftResult`, `ValidationResult`, `CheckResult`, `GateResult`, and per-column result views
@@ -31,13 +33,15 @@ Development continues on the development branches. Changes intended for the next
 - Optional lazy DuckDB relation source with exact count/schema metadata, projection pushdown, Arrow record batches, and full pandas materialization only for exact/full-row APIs
 - Optional `duckdb` dependency group containing DuckDB plus its Arrow transport
 - Dedicated Arrow/DuckDB interoperability CI coverage
-- Execution provenance that distinguishes exact full-stream metrics, bounded row samples, estimates, and full materialization
+- Versioned execution-provenance schema v1 with shared `method`, `full_materialization`, source, sampling, strategy, component, and reason fields while preserving legacy operation-specific metadata during the `0.x` migration
 - Optional Rust native core and Python bridge for accelerated streaming numeric state, cardinality/quantile sketches, and backend routing
 - Dedicated Arrow/NumPy-fallback and Rust/native CI coverage
 - Reproducible performance benchmark workflow with time/RSS summaries and retained JSON artifacts
 - PEP 561 `py.typed` marker for downstream type-checking support
 - Optional `excel` dependency group for XLS/XLSX readers
 - Optional `plot` dependency group for Matplotlib/Seaborn chart and report plotting support
+- Optional `docs` dependency group plus a MkDocs Material documentation site covering source-aware execution, execution provenance, quality gates, custom checks, and plugin trust boundaries
+- Strict path-scoped documentation CI using `mkdocs build --strict`
 - Contributor architecture guidance covering canonical layers, streaming provenance, optional-dependency boundaries, and performance-sensitive changes
 - Focused examples for domain-specific quality gates and persistent snapshot monitoring
 
@@ -51,7 +55,8 @@ Development continues on the development branches. Changes intended for the next
 - Arbitrary custom Python checks explicitly remain exact and materialize non-pandas sources rather than silently sampling user-defined invariants
 - Top-level gate execution now reports whether any selected check family required full materialization
 - Materialization provenance is now based on actual execution semantics rather than assuming only file-backed sources can materialize
-- `framevitals.api` is now a lazy compatibility facade over the canonical focused, analysis, planning, operations, check, and plugin engines instead of a second eager implementation
+- Focused statistics, anomaly, relationship, role, health, ML-readiness, quality, drift, validation, custom-check, and gate paths are converging on execution-provenance schema v1 without removing legacy metadata keys
+- `framevitals.api` is now a lazy compatibility facade over the canonical focused, analysis, planning, operations, check, plugin, and source-inspection engines instead of a second eager implementation
 - The installed CLI now routes directly through canonical APIs and no longer relies on a runtime monkeypatch shim
 - CLI `--version` works from a no-dependencies wheel smoke install without importing the analytics stack
 - The `all` extra now includes Arrow and DuckDB interoperability capabilities
@@ -61,6 +66,7 @@ Development continues on the development branches. Changes intended for the next
 - Flask startup no longer eagerly imports the agentic AI or report/plot stacks; those capabilities load only when their endpoints are used
 - Excel loading now reports a FrameVitals-specific install hint when the optional reader capability is missing
 - README and roadmap now describe the source-aware analyze → compare → validate → gate → monitor direction, source interoperability, exact-vs-bounded execution, optional capability groups, plugins, and the reusable Gate Action
+- Project metadata now advertises the package as typed and points documentation users at the maintained `docs/` tree
 - Package CI now requires the `py.typed` marker to be present in built wheels
 - Test and package workflows now use read-only default permissions and cancel stale runs on the same branch
 - Release publishing now verifies that the GitHub release tag, `pyproject.toml` version, and `framevitals.__version__` match before building/publishing

From a8c3889d2b8b1701d25cad87bb5678b2d996adb5 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:51:28 +0530
Subject: [PATCH 318/439] feat: add dict-compatible focused diagnostic results

---
 src/framevitals/result.py | 123 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 120 insertions(+), 3 deletions(-)

diff --git a/src/framevitals/result.py b/src/framevitals/result.py
index 683083e..95e7b9b 100644
--- a/src/framevitals/result.py
+++ b/src/framevitals/result.py
@@ -1,8 +1,8 @@
 """Public result objects for FrameVitals.
 
-``AnalysisResult`` intentionally subclasses :class:`dict` during the 0.x series.
-That preserves existing mapping behaviour and JSON compatibility while adding a
-more discoverable object-oriented API for notebooks, applications, and reports.
+FrameVitals result objects intentionally subclass :class:`dict` during the 0.x
+series. That preserves existing mapping behaviour and JSON compatibility while
+adding discoverable helpers for notebooks, applications, reports, and CI.
 """
 
 from __future__ import annotations
@@ -31,6 +31,123 @@ def __getattr__(self, name: str) -> Any:
             raise AttributeError(name) from exc
 
 
+class DiagnosticResult(dict):
+    """Dict-compatible result returned by focused diagnostic APIs.
+
+    The diagnostic name is stored as Python-side metadata rather than inserted
+    into the mapping, so wrapping an existing focused payload does not mutate its
+    JSON schema during the 0.x compatibility window.
+    """
+
+    def __init__(
+        self,
+        *args,
+        diagnostic: str = "diagnostic",
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+        self._diagnostic = str(diagnostic or "diagnostic")
+
+    def __getattr__(self, name: str) -> Any:
+        try:
+            return self[name]
+        except KeyError as exc:
+            raise AttributeError(name) from exc
+
+    @property
+    def diagnostic(self) -> str:
+        return self._diagnostic
+
+    @property
+    def dataset_name(self) -> str | None:
+        value = self.get("dataset_name")
+        return str(value) if value is not None else None
+
+    @property
+    def execution(self) -> dict[str, Any]:
+        value = self.get("execution", {})
+        return value if isinstance(value, dict) else {}
+
+    @property
+    def source(self) -> dict[str, Any]:
+        value = self.get("source")
+        if not isinstance(value, dict):
+            value = self.execution.get("source")
+        if not isinstance(value, dict):
+            value = self.get("source_metadata")
+        return value if isinstance(value, dict) else {}
+
+    @property
+    def available(self) -> bool:
+        value = self.get("available")
+        return True if value is None else bool(value)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a detached plain dictionary without Python-side metadata."""
+        return deepcopy(dict(self))
+
+    def to_json(
+        self,
+        destination: str | Path | None = None,
+        *,
+        indent: int = 2,
+    ) -> str | Path:
+        """Serialize the complete diagnostic result or write it to a file."""
+        rendered = json.dumps(self.to_dict(), indent=indent, default=str)
+        if destination is None:
+            return rendered
+        path = Path(destination)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(rendered + "\n", encoding="utf-8")
+        return path
+
+    def summary(self) -> dict[str, Any]:
+        """Return a compact operation-agnostic execution summary."""
+        execution = self.execution
+        shape = self.get("shape")
+        if not isinstance(shape, dict):
+            shape = {}
+        source = self.source
+        return {
+            "diagnostic": self.diagnostic,
+            "dataset_name": self.dataset_name,
+            "available": self.available,
+            "shape": dict(shape),
+            "method": execution.get("method"),
+            "full_materialization": execution.get("full_materialization"),
+            "sampled": execution.get("sampled"),
+            "source_rows": execution.get("source_rows", source.get("rows")),
+            "source_columns": execution.get("source_columns", source.get("columns")),
+            "sample_rows": execution.get("sample_rows"),
+            "execution_schema_version": execution.get("execution_schema_version"),
+        }
+
+    def summary_text(self) -> str:
+        """Render a compact terminal-friendly focused-diagnostic summary."""
+        summary = self.summary()
+        lines = [
+            f"FrameVitals {self.diagnostic}",
+            f"Dataset         {summary.get('dataset_name') or 'unknown'}",
+            f"Available       {'yes' if summary.get('available') else 'no'}",
+        ]
+        if summary.get("method") is not None:
+            lines.append(f"Method          {summary['method']}")
+        if summary.get("source_rows") is not None:
+            lines.append(f"Source rows     {summary['source_rows']}")
+        if summary.get("sample_rows") is not None:
+            lines.append(f"Sample rows     {summary['sample_rows']}")
+        if summary.get("sampled") is not None:
+            lines.append(
+                f"Sampled         {'yes' if summary.get('sampled') else 'no'}"
+            )
+        if summary.get("full_materialization") is not None:
+            lines.append(
+                "Materialized    "
+                f"{'yes' if summary.get('full_materialization') else 'no'}"
+            )
+        return "\n".join(lines)
+
+
 class AnalysisResult(dict):
     """Backward-compatible result returned by :func:`framevitals.analyze`."""
 

From ba9052201ed17f684fdc01cc299e8810a61f3527 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:52:20 +0530
Subject: [PATCH 319/439] feat: return focused DiagnosticResult objects

---
 src/framevitals/focused.py | 109 ++++++++++++++++++++++++-------------
 1 file changed, 71 insertions(+), 38 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index eb79201..abb2250 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -17,6 +17,7 @@
     load_fully_materializes,
     normalize_execution,
 )
+from framevitals.result import DiagnosticResult
 from framevitals.sources import StreamingDatasetSource, resolve_source
 
 
@@ -36,26 +37,42 @@ def _load(data: DataInput, *, label: str = "Dataset") -> tuple[pd.DataFrame, str
     return dataframe, metadata.name
 
 
-def _named(payload: dict[str, Any], source_name: str) -> dict[str, Any]:
-    return {"dataset_name": source_name, **payload}
+def _named(
+    payload: dict[str, Any],
+    source_name: str,
+    *,
+    diagnostic: str,
+) -> DiagnosticResult:
+    return DiagnosticResult(
+        {"dataset_name": source_name, **payload},
+        diagnostic=diagnostic,
+    )
 
 
-def profile(data: DataInput) -> dict[str, Any]:
+def profile(data: DataInput) -> DiagnosticResult:
     """Profile a dataset, streaming Arrow-capable file sources when available."""
     source = resolve_source(data)
     metadata = source.inspect()
     if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
         from framevitals.streaming_profile import build_streaming_profile
 
-        return _named(build_streaming_profile(source), metadata.name)
+        return _named(
+            build_streaming_profile(source),
+            metadata.name,
+            diagnostic="profile",
+        )
 
     from framevitals.profiler import build_profile
 
     dataframe = source.load()
-    return _named(build_profile(dataframe), metadata.name)
+    return _named(
+        build_profile(dataframe),
+        metadata.name,
+        diagnostic="profile",
+    )
 
 
-def roles(data: DataInput) -> dict[str, Any]:
+def roles(data: DataInput) -> DiagnosticResult:
     source = resolve_source(data)
     metadata = source.inspect()
     if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
@@ -68,20 +85,23 @@ def roles(data: DataInput) -> dict[str, Any]:
             return_sample=True,
         )
         payload = infer_streaming_column_roles(sample, profile=dataset_profile)
-        return _named(payload, metadata.name)
+        return _named(payload, metadata.name, diagnostic="roles")
 
     from framevitals.column_roles import infer_column_roles, summarize_roles
 
     dataframe = source.load()
     column_roles = infer_column_roles(dataframe)
-    return {
-        "dataset_name": metadata.name,
-        "columns": column_roles,
-        "summary": summarize_roles(column_roles),
-    }
+    return _named(
+        {
+            "columns": column_roles,
+            "summary": summarize_roles(column_roles),
+        },
+        metadata.name,
+        diagnostic="roles",
+    )
 
 
-def health(data: DataInput) -> dict[str, Any]:
+def health(data: DataInput) -> DiagnosticResult:
     source = resolve_source(data)
     metadata = source.inspect()
     if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
@@ -90,17 +110,21 @@ def health(data: DataInput) -> dict[str, Any]:
 
         dataset_profile, sample = build_streaming_profile(source, return_sample=True)
         payload = calculate_health_score_from_profile_sample(dataset_profile, sample)
-        return _named(payload, metadata.name)
+        return _named(payload, metadata.name, diagnostic="health")
 
     from framevitals.health_score import calculate_health_score
     from framevitals.profiler import build_profile
 
     dataframe = source.load()
     dataset_profile = build_profile(dataframe)
-    return _named(calculate_health_score(dataframe, dataset_profile), metadata.name)
+    return _named(
+        calculate_health_score(dataframe, dataset_profile),
+        metadata.name,
+        diagnostic="health",
+    )
 
 
-def ml_readiness(data: DataInput) -> dict[str, Any]:
+def ml_readiness(data: DataInput) -> DiagnosticResult:
     source = resolve_source(data)
     metadata = source.inspect()
     if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
@@ -108,7 +132,11 @@ def ml_readiness(data: DataInput) -> dict[str, Any]:
         from framevitals.streaming_profile import build_streaming_profile
 
         dataset_profile = build_streaming_profile(source)
-        return _named(calculate_ml_readiness_from_profile(dataset_profile), metadata.name)
+        return _named(
+            calculate_ml_readiness_from_profile(dataset_profile),
+            metadata.name,
+            diagnostic="ml_readiness",
+        )
 
     from framevitals.ml_readiness import calculate_ml_readiness
     from framevitals.profiler import build_profile
@@ -118,6 +146,7 @@ def ml_readiness(data: DataInput) -> dict[str, Any]:
     return _named(
         calculate_ml_readiness(dataframe, profile=dataset_profile),
         metadata.name,
+        diagnostic="ml_readiness",
     )
 
 
@@ -127,7 +156,7 @@ def quality(
     max_sample_rows: int = 5_000,
     max_columns: int = 100,
     max_missingness_columns: int = 25,
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     source = resolve_source(data)
     metadata = source.inspect()
     if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
@@ -148,7 +177,7 @@ def quality(
             max_columns=max_columns,
             max_missingness_columns=max_missingness_columns,
         )
-        return _named(payload, metadata.name)
+        return _named(payload, metadata.name, diagnostic="quality")
 
     from framevitals.column_roles import infer_column_roles
     from framevitals.profiler import build_profile
@@ -165,7 +194,7 @@ def quality(
         max_columns=max_columns,
         max_missingness_columns=max_missingness_columns,
     )
-    return _named(payload, metadata.name)
+    return _named(payload, metadata.name, diagnostic="quality")
 
 
 def statistics(
@@ -173,7 +202,7 @@ def statistics(
     *,
     max_pairs: int = 20,
     mode: str = "standard",
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     """Run deep statistics through the same large-data budget as full analysis."""
     from framevitals.budgeted_analysis import run_budgeted_deep_statistics
     from framevitals.execution import derive_execution_budget
@@ -229,7 +258,7 @@ def statistics(
             source=metadata.to_dict(),
         )
         payload["source"] = metadata.to_dict()
-        return _named(payload, metadata.name)
+        return _named(payload, metadata.name, diagnostic="statistics")
 
     dataframe = source.load()
     budget = derive_execution_budget(
@@ -249,7 +278,7 @@ def statistics(
         source=metadata.to_dict(),
     )
     payload["source"] = metadata.to_dict()
-    return _named(payload, metadata.name)
+    return _named(payload, metadata.name, diagnostic="statistics")
 
 
 def anomalies(
@@ -260,7 +289,7 @@ def anomalies(
     max_columns: int = 30,
     top_k: int = 25,
     mode: str = "standard",
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     """Run anomaly diagnostics with bounded covariance/neighbor work."""
     from framevitals.budgeted_analysis import run_budgeted_anomalies
     from framevitals.execution import derive_execution_budget
@@ -292,12 +321,16 @@ def anomalies(
                 scope="bounded_anomaly_detection",
                 extra={"projected_columns": 0},
             )
-            return _named({
-                "available": False,
-                "reason": "No numeric columns available for anomaly analysis.",
-                "execution": execution,
-                "source": metadata.to_dict(),
-            }, metadata.name)
+            return _named(
+                {
+                    "available": False,
+                    "reason": "No numeric columns available for anomaly analysis.",
+                    "execution": execution,
+                    "source": metadata.to_dict(),
+                },
+                metadata.name,
+                diagnostic="anomalies",
+            )
 
         sample_limit = max(100, int(budget.anomaly_sample_rows))
         sample = sample_streaming_source(
@@ -341,7 +374,7 @@ def anomalies(
             source=metadata.to_dict(),
         )
         payload["source"] = metadata.to_dict()
-        return _named(payload, metadata.name)
+        return _named(payload, metadata.name, diagnostic="anomalies")
 
     dataframe = source.load()
     budget = derive_execution_budget(
@@ -364,7 +397,7 @@ def anomalies(
         source=metadata.to_dict(),
     )
     payload["source"] = metadata.to_dict()
-    return _named(payload, metadata.name)
+    return _named(payload, metadata.name, diagnostic="anomalies")
 
 
 def relationships(
@@ -375,7 +408,7 @@ def relationships(
     min_abs_correlation: float = 0.80,
     max_candidate_pairs: int = 250_000,
     max_edges_returned: int = 5_000,
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     from framevitals.relationship_graph import build_numeric_relationship_graph
 
     source = resolve_source(data)
@@ -431,7 +464,7 @@ def relationships(
             },
             extra={"projected_columns": int(len(numeric_columns))},
         )
-        return _named(payload, metadata.name)
+        return _named(payload, metadata.name, diagnostic="relationships")
 
     dataframe = source.load()
     payload = build_numeric_relationship_graph(
@@ -463,21 +496,21 @@ def relationships(
             "relationship_candidates": "bounded_row_sample" if sampled else "full_input",
         },
     )
-    return _named(payload, metadata.name)
+    return _named(payload, metadata.name, diagnostic="relationships")
 
 
 def target_analysis(
     data: DataInput,
     *,
     target: str,
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     source = resolve_source(data)
     metadata = source.inspect()
     if metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
         from framevitals.streaming_target import run_streaming_target_analysis
 
         payload = run_streaming_target_analysis(source, target=target)
-        return _named(payload, metadata.name)
+        return _named(payload, metadata.name, diagnostic="target_analysis")
 
     from framevitals.column_roles import infer_column_roles
     from framevitals.target_intelligence import run_target_intelligence
@@ -491,4 +524,4 @@ def target_analysis(
         target_column=target,
         column_roles=column_roles,
     )
-    return _named(payload, metadata.name)
+    return _named(payload, metadata.name, diagnostic="target_analysis")

From eb877798a24ec454187d6632fffc14a6b762f780 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:52:54 +0530
Subject: [PATCH 320/439] typing: expose focused DiagnosticResult type

---
 src/framevitals/__init__.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index 4ee7234..ed6b8fc 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -12,7 +12,7 @@
     GateResult,
     ValidationResult,
 )
-from framevitals.result import AnalysisResult, ColumnResult
+from framevitals.result import AnalysisResult, ColumnResult, DiagnosticResult
 from framevitals.snapshots import (
     AnalysisSnapshot,
     SnapshotHistory,
@@ -48,28 +48,28 @@ def inspect_source(data: Any) -> dict[str, Any]:
     return _inspect_source(data)
 
 
-def profile(data: Any) -> dict[str, Any]:
+def profile(data: Any) -> DiagnosticResult:
     """Profile shape, dtypes, missingness, cardinality, and basic summaries."""
     from framevitals.focused import profile as _profile
 
     return _profile(data)
 
 
-def roles(data: Any) -> dict[str, Any]:
+def roles(data: Any) -> DiagnosticResult:
     """Infer semantic and structural roles for dataset columns."""
     from framevitals.focused import roles as _roles
 
     return _roles(data)
 
 
-def health(data: Any) -> dict[str, Any]:
+def health(data: Any) -> DiagnosticResult:
     """Calculate only the FrameVitals data-health score."""
     from framevitals.focused import health as _health
 
     return _health(data)
 
 
-def ml_readiness(data: Any) -> dict[str, Any]:
+def ml_readiness(data: Any) -> DiagnosticResult:
     """Calculate only ML-readiness diagnostics."""
     from framevitals.focused import ml_readiness as _ml_readiness
 
@@ -82,7 +82,7 @@ def quality(
     max_sample_rows: int = 5_000,
     max_columns: int = 100,
     max_missingness_columns: int = 25,
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     """Run practical deterministic data-quality diagnostics only."""
     from framevitals.focused import quality as _quality
 
@@ -99,7 +99,7 @@ def statistics(
     *,
     max_pairs: int = 20,
     mode: str = "standard",
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     """Run the deep statistical diagnostics layer with adaptive row budgets."""
     from framevitals.focused import statistics as _statistics
 
@@ -114,7 +114,7 @@ def anomalies(
     max_columns: int = 30,
     top_k: int = 25,
     mode: str = "standard",
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     """Run only the bounded FrameVitals tabular anomaly ensemble."""
     from framevitals.focused import anomalies as _anomalies
 
@@ -136,7 +136,7 @@ def relationships(
     min_abs_correlation: float = 0.80,
     max_candidate_pairs: int = 250_000,
     max_edges_returned: int = 5_000,
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     """Discover strong numeric relationships without a dense correlation matrix."""
     from framevitals.focused import relationships as _relationships
 
@@ -157,7 +157,7 @@ def system_info(*, probe_gpu: bool = True) -> dict[str, Any]:
     return _system_info(probe_gpu=probe_gpu)
 
 
-def target_analysis(data: Any, *, target: str) -> dict[str, Any]:
+def target_analysis(data: Any, *, target: str) -> DiagnosticResult:
     """Run target quality, leakage, association, and split diagnostics only."""
     from framevitals.focused import target_analysis as _target_analysis
 
@@ -343,6 +343,7 @@ def gate(
     "SnapshotHistory",
     "CleaningPlan",
     "ColumnResult",
+    "DiagnosticResult",
     "DataCheck",
     "CheckResult",
     "DriftResult",

From 86805b27b992b3468f0bbf8be2a5d89f885b2b5e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:53:08 +0530
Subject: [PATCH 321/439] test: cover focused DiagnosticResult ergonomics

---
 tests/test_diagnostic_results.py | 90 ++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)
 create mode 100644 tests/test_diagnostic_results.py

diff --git a/tests/test_diagnostic_results.py b/tests/test_diagnostic_results.py
new file mode 100644
index 0000000..08bb938
--- /dev/null
+++ b/tests/test_diagnostic_results.py
@@ -0,0 +1,90 @@
+import json
+
+import numpy as np
+import pandas as pd
+
+import framevitals as fv
+
+
+def _frame(rows: int = 120) -> pd.DataFrame:
+    values = np.arange(rows, dtype=np.float64)
+    return pd.DataFrame({
+        "value": values,
+        "other": values * 2.0 + 1.0,
+        "group": [f"g-{index % 4}" for index in range(rows)],
+        "target": [index % 2 for index in range(rows)],
+    })
+
+
+def test_profile_returns_dict_compatible_diagnostic_result_without_schema_mutation(tmp_path):
+    result = fv.profile(_frame())
+
+    assert isinstance(result, dict)
+    assert isinstance(result, fv.DiagnosticResult)
+    assert result.diagnostic == "profile"
+    assert result.dataset_name == "<dataframe>"
+    assert result.available is True
+    assert "diagnostic" not in result
+
+    detached = result.to_dict()
+    assert type(detached) is dict
+    assert detached == dict(result)
+    detached["shape"]["rows"] = -1
+    assert result["shape"]["rows"] == 120
+
+    rendered = json.loads(result.to_json())
+    assert rendered == dict(result)
+    assert "diagnostic" not in rendered
+
+    destination = tmp_path / "profile.json"
+    returned = result.to_json(destination)
+    assert returned == destination
+    assert json.loads(destination.read_text(encoding="utf-8")) == dict(result)
+
+
+def test_focused_apis_label_their_diagnostic_result():
+    frame = _frame()
+
+    results = {
+        "roles": fv.roles(frame),
+        "health": fv.health(frame),
+        "ml_readiness": fv.ml_readiness(frame),
+        "quality": fv.quality(frame),
+        "statistics": fv.statistics(frame, mode="quick", max_pairs=2),
+        "anomalies": fv.anomalies(frame, mode="quick", max_columns=3, top_k=5),
+        "relationships": fv.relationships(frame, max_sample_rows=64),
+        "target_analysis": fv.target_analysis(frame, target="target"),
+    }
+
+    for diagnostic, result in results.items():
+        assert isinstance(result, fv.DiagnosticResult)
+        assert result.diagnostic == diagnostic
+        assert result.dataset_name == "<dataframe>"
+        assert "diagnostic" not in result
+
+
+def test_diagnostic_result_execution_and_source_helpers_follow_public_provenance():
+    result = fv.statistics(_frame(800), mode="quick", max_pairs=2)
+
+    assert result.execution["execution_schema_version"] == "1"
+    assert result.execution["method"] == "bounded_deep_statistics"
+    assert result.source["format"] == "pandas"
+    assert result.source["rows"] == 800
+
+    summary = result.summary()
+    assert summary["diagnostic"] == "statistics"
+    assert summary["dataset_name"] == "<dataframe>"
+    assert summary["method"] == "bounded_deep_statistics"
+    assert summary["execution_schema_version"] == "1"
+    assert summary["source_rows"] == 800
+    assert "FrameVitals statistics" in result.summary_text()
+
+
+def test_unavailable_diagnostic_result_exposes_available_false():
+    frame = pd.DataFrame({"label": ["a", "b", "c", "d"]})
+    result = fv.anomalies(frame, mode="quick")
+
+    assert isinstance(result, fv.DiagnosticResult)
+    assert result.available is False
+    assert result["available"] is False
+    assert result.execution["method"] == "bounded_anomaly_detection"

From 4d142950e9ad63a9131b3d882afe41260e6d400e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:54:23 +0530
Subject: [PATCH 322/439] docs: document public result objects

---
 docs/result-objects.md | 169 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 169 insertions(+)
 create mode 100644 docs/result-objects.md

diff --git a/docs/result-objects.md b/docs/result-objects.md
new file mode 100644
index 0000000..c23b6f2
--- /dev/null
+++ b/docs/result-objects.md
@@ -0,0 +1,169 @@
+# Result objects
+
+FrameVitals public workflows return **dict-compatible result objects** during the
+`0.x` series. This keeps existing mapping/JSON code working while adding discoverable
+helpers for notebooks, applications, reports, and CI.
+
+## Why dict-compatible objects?
+
+A FrameVitals result should be easy to:
+
+- index like a normal dictionary;
+- serialize to JSON;
+- pass into existing application code;
+- inspect interactively with named properties and helper methods.
+
+The `0.x` series therefore favors additive wrappers over a hard break to dataclasses
+or validation models.
+
+## Full analysis
+
+`fv.analyze(...)` returns `AnalysisResult`.
+
+```python
+report = fv.analyze(data)
+
+report.health
+report.ml_readiness
+report.findings
+report.recommendations
+report.column("age")
+report.summary()
+report.summary_text()
+report.to_json("analysis.json")
+report.to_html("analysis.html")
+report.snapshot("snapshot.json")
+```
+
+`AnalysisResult` remains a `dict`, so existing code still works:
+
+```python
+score = report["health"]["overall_score"]
+```
+
+## Focused diagnostics
+
+Focused APIs return `DiagnosticResult`:
+
+- `fv.profile(...)`
+- `fv.roles(...)`
+- `fv.health(...)`
+- `fv.ml_readiness(...)`
+- `fv.quality(...)`
+- `fv.statistics(...)`
+- `fv.anomalies(...)`
+- `fv.relationships(...)`
+- `fv.target_analysis(...)`
+
+```python
+stats = fv.statistics(data, mode="quick")
+
+stats.diagnostic       # "statistics"
+stats.dataset_name
+stats.execution
+stats.source
+stats.available
+stats.summary()
+stats.summary_text()
+stats.to_json("statistics.json")
+```
+
+The diagnostic label is Python-side metadata and is **not injected into the mapping**.
+That means wrapping a focused payload does not mutate its JSON schema:
+
+```python
+assert "diagnostic" not in stats
+assert dict(stats) == stats.to_dict()
+```
+
+`to_dict()` returns a detached deep copy so callers can modify it without mutating the
+live result object.
+
+## Quality operations
+
+### DriftResult
+
+`fv.compare(...)` returns `DriftResult` with conveniences such as:
+
+```python
+drift.severity
+drift.status
+drift.columns
+drift.summary_text()
+drift.to_json()
+```
+
+### ValidationResult
+
+`fv.validate(...)` returns `ValidationResult`:
+
+```python
+validation.valid
+validation.status
+validation.findings
+validation.summary_text()
+```
+
+### CheckResult
+
+`fv.run_checks(...)` returns `CheckResult`:
+
+```python
+checks.status
+checks.passed
+checks.results
+checks.findings
+checks.summary_text()
+```
+
+### GateResult
+
+`fv.gate(...)` returns `GateResult`:
+
+```python
+gate.status
+gate.passed
+gate.checks_run
+gate.reasons
+gate.summary_text()
+```
+
+All four quality result types remain mapping-compatible and expose `to_dict()` /
+`to_json()` through the shared quality-result base.
+
+## Monitoring state
+
+`AnalysisSnapshot` is a compact dict-compatible monitoring record derived from an
+`AnalysisResult`. `SnapshotHistory` manages persisted snapshots without storing raw
+input datasets.
+
+```python
+history = fv.SnapshotHistory(".framevitals/history")
+history.add(report.snapshot(), label="production")
+change = history.compare_latest()
+```
+
+## Execution metadata
+
+Focused and quality results increasingly expose the shared execution-provenance
+contract under `result.execution` or `result["execution"]`.
+
+```python
+execution = stats.execution
+print(execution["execution_schema_version"])
+print(execution["method"])
+print(execution["sampled"])
+print(execution["full_materialization"])
+```
+
+See [Execution provenance](execution-provenance.md) for the schema and exactness rules.
+
+## Compatibility policy during 0.x
+
+FrameVitals aims to evolve result ergonomics without needless churn:
+
+1. result objects remain subclasses of `dict`;
+2. helper properties should not silently change serialized payloads;
+3. new shared metadata is added deliberately and versioned where appropriate;
+4. legacy operation-specific fields remain available during migration;
+5. breaking result-schema changes must be documented before release.

From f563d7b6bddf32cadb854fb54d4b2e8c84cd86b3 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:54:32 +0530
Subject: [PATCH 323/439] test: lock public package surface

---
 tests/test_public_surface_contract.py | 64 +++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 tests/test_public_surface_contract.py

diff --git a/tests/test_public_surface_contract.py b/tests/test_public_surface_contract.py
new file mode 100644
index 0000000..699a920
--- /dev/null
+++ b/tests/test_public_surface_contract.py
@@ -0,0 +1,64 @@
+import framevitals as fv
+
+
+def test_core_public_surface_is_exported():
+    expected = {
+        "AnalysisConfig",
+        "AnalysisPlan",
+        "AnalysisResult",
+        "AnalysisSnapshot",
+        "SnapshotHistory",
+        "CleaningPlan",
+        "ColumnResult",
+        "DiagnosticResult",
+        "DataCheck",
+        "CheckResult",
+        "DriftResult",
+        "GateResult",
+        "ValidationResult",
+        "inspect_source",
+        "profile",
+        "roles",
+        "health",
+        "ml_readiness",
+        "quality",
+        "statistics",
+        "anomalies",
+        "relationships",
+        "system_info",
+        "target_analysis",
+        "analyze",
+        "plan",
+        "plan_cleaning",
+        "clean",
+        "compare",
+        "infer_contract",
+        "validate",
+        "check",
+        "run_checks",
+        "discover_checks",
+        "gate",
+        "available_modules",
+        "create_snapshot",
+        "load_snapshot",
+        "compare_snapshots",
+        "__version__",
+    }
+
+    assert expected.issubset(set(fv.__all__))
+    for name in expected:
+        assert hasattr(fv, name), name
+
+
+def test_result_types_remain_dict_compatible():
+    for result_type in (
+        fv.AnalysisResult,
+        fv.AnalysisSnapshot,
+        fv.ColumnResult,
+        fv.DiagnosticResult,
+        fv.CheckResult,
+        fv.DriftResult,
+        fv.GateResult,
+        fv.ValidationResult,
+    ):
+        assert issubclass(result_type, dict)

From dab75ffaeb3051094c856e9d13f60479a297e95f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:57:23 +0530
Subject: [PATCH 324/439] test: cover zero-dependency Polars interoperability

---
 tests/test_polars_interop.py | 87 ++++++++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 tests/test_polars_interop.py

diff --git a/tests/test_polars_interop.py b/tests/test_polars_interop.py
new file mode 100644
index 0000000..450c1da
--- /dev/null
+++ b/tests/test_polars_interop.py
@@ -0,0 +1,87 @@
+import pytest
+
+pytest.importorskip("pyarrow")
+pl = pytest.importorskip("polars")
+
+import framevitals as fv
+from framevitals.sources import ArrowTableSource, resolve_source
+
+
+def _frame(rows: int = 6_000):
+    return pl.DataFrame({
+        "value": [float(index) for index in range(rows)],
+        "other": [float(index * 2) for index in range(rows)],
+        "group": [f"g-{index % 5}" for index in range(rows)],
+    })
+
+
+def test_polars_dataframe_uses_generic_arrow_capsule_source():
+    frame = _frame(2_000)
+
+    source = resolve_source(frame)
+    metadata = source.inspect()
+
+    assert isinstance(source, ArrowTableSource)
+    assert metadata.name == "<arrow_capsule_table>"
+    assert metadata.kind == "memory"
+    assert metadata.format == "arrow"
+    assert metadata.rows == frame.height
+    assert metadata.columns == frame.width
+    assert metadata.supports_projection is True
+    assert metadata.supports_streaming is True
+
+    info = fv.inspect_source(frame)
+    assert info == metadata.to_dict()
+
+
+def test_polars_profile_stays_on_arrow_batch_path(monkeypatch):
+    frame = _frame(12_000)
+
+    def fail_load(self):
+        raise AssertionError("Polars profiling must not materialize through pandas")
+
+    monkeypatch.setattr(ArrowTableSource, "load", fail_load)
+    result = fv.profile(frame)
+
+    assert isinstance(result, fv.DiagnosticResult)
+    assert result.diagnostic == "profile"
+    assert result.dataset_name == "<arrow_capsule_table>"
+    assert result["shape"] == {"rows": frame.height, "columns": frame.width}
+    assert result["streaming_metadata"]["enabled"] is True
+    assert result["streaming_metadata"]["full_materialization"] is False
+    assert result["source_metadata"]["format"] == "arrow"
+
+
+def test_polars_quick_analysis_stays_on_streaming_path(monkeypatch):
+    frame = _frame(6_000)
+
+    def fail_load(self):
+        raise AssertionError("Polars analysis must not materialize through pandas")
+
+    monkeypatch.setattr(ArrowTableSource, "load", fail_load)
+    result = fv.analyze(frame, mode="quick", artifacts=False, workers=1)
+
+    assert result["filename"] == "<arrow_capsule_table>"
+    assert result["profile"]["shape"] == {
+        "rows": frame.height,
+        "columns": frame.width,
+    }
+    assert result["execution"]["streaming"]["enabled"] is True
+    assert result["execution"]["streaming"]["full_materialization"] is False
+
+
+def test_polars_bounded_statistics_do_not_load_full_dataframe(monkeypatch):
+    frame = _frame(20_000)
+
+    def fail_load(self):
+        raise AssertionError("Polars statistics must remain on bounded Arrow batches")
+
+    monkeypatch.setattr(ArrowTableSource, "load", fail_load)
+    result = fv.statistics(frame, mode="quick", max_pairs=2)
+
+    assert isinstance(result, fv.DiagnosticResult)
+    assert result.execution["execution_schema_version"] == "1"
+    assert result.execution["method"] == "bounded_deep_statistics"
+    assert result.execution["full_materialization"] is False
+    assert result.execution["source_rows"] == frame.height
+    assert result.execution["sampled"] is True

From b07989b884e957525eb511b18b3b0cc75e6142e4 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:57:35 +0530
Subject: [PATCH 325/439] ci: verify Polars Arrow interoperability

---
 .github/workflows/polars-interop.yml | 61 ++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 .github/workflows/polars-interop.yml

diff --git a/.github/workflows/polars-interop.yml b/.github/workflows/polars-interop.yml
new file mode 100644
index 0000000..6e8aefa
--- /dev/null
+++ b/.github/workflows/polars-interop.yml
@@ -0,0 +1,61 @@
+name: Polars Interoperability
+
+on:
+  push:
+    branches: [main, dev, "develop/august"]
+    paths:
+      - "src/framevitals/sources.py"
+      - "src/framevitals/analysis_api.py"
+      - "src/framevitals/focused.py"
+      - "src/framevitals/result.py"
+      - "tests/test_arrow_memory.py"
+      - "tests/test_polars_interop.py"
+      - ".github/workflows/polars-interop.yml"
+  pull_request:
+    paths:
+      - "src/framevitals/sources.py"
+      - "src/framevitals/analysis_api.py"
+      - "src/framevitals/focused.py"
+      - "src/framevitals/result.py"
+      - "tests/test_arrow_memory.py"
+      - "tests/test_polars_interop.py"
+      - ".github/workflows/polars-interop.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  polars-arrow:
+    name: Polars via Arrow C Stream (Python 3.12)
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install FrameVitals Arrow capability and Polars
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[arrow,dev]"
+          pip install "polars>=1,<2"
+
+      - name: Verify dependency consistency
+        run: python -m pip check
+
+      - name: Run Polars interoperability tests
+        env:
+          FRAMEVITALS_BACKEND: numpy
+        run: pytest -v tests/test_arrow_memory.py tests/test_polars_interop.py

From 728ec2b7355719c6803cdefdc04187bbfb4d92b3 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:58:13 +0530
Subject: [PATCH 326/439] docs: add extension author guide

---
 docs/extending-framevitals.md | 163 ++++++++++++++++++++++++++++++++++
 1 file changed, 163 insertions(+)
 create mode 100644 docs/extending-framevitals.md

diff --git a/docs/extending-framevitals.md b/docs/extending-framevitals.md
new file mode 100644
index 0000000..6675782
--- /dev/null
+++ b/docs/extending-framevitals.md
@@ -0,0 +1,163 @@
+# Extending FrameVitals
+
+FrameVitals is designed to be extended at the edges rather than forked in the core.
+There are two intentionally small extension boundaries:
+
+1. **custom data checks** for domain invariants;
+2. **`DatasetSource` implementations** for new storage/query systems.
+
+Third-party check packages can additionally register checks through standard Python
+entry points. Source adapters are normal Python objects implementing the source
+protocol and can be shipped by any package without FrameVitals importing that package.
+
+## Custom checks
+
+Use `@framevitals.check` for application-owned invariants:
+
+```python
+import framevitals as fv
+
+@fv.check(
+    "positive revenue",
+    severity="error",
+    description="Revenue cannot be negative.",
+)
+def positive_revenue(df):
+    minimum = float(df["revenue"].min())
+    return {
+        "passed": minimum >= 0,
+        "message": "Negative revenue found." if minimum < 0 else "Revenue is valid.",
+        "details": {"minimum": minimum},
+    }
+```
+
+A check may return either a boolean-like value or a mapping containing `passed`.
+Mappings can also include `message` and `details`.
+
+Custom checks intentionally run against the complete DataFrame. FrameVitals cannot
+safely infer whether an arbitrary Python invariant is sampleable, so it does not
+silently weaken user-defined rules to bounded samples.
+
+## Publishing a check plugin
+
+A third-party package can expose checks through the `framevitals.checks` entry-point
+group:
+
+```toml
+[project.entry-points."framevitals.checks"]
+positive_revenue = "acme_framevitals_checks:positive_revenue"
+```
+
+The exported object can be a `DataCheck` or a compatible DataFrame callable.
+Applications opt in explicitly:
+
+```python
+checks = fv.discover_checks()
+result = fv.gate(current, custom_checks=checks)
+```
+
+FrameVitals never auto-loads installed check plugins. Loading an entry point executes
+provider code, so discovery is an explicit trust decision.
+
+### Plugin guidance
+
+A good provider package should:
+
+- keep public check names stable;
+- avoid network or filesystem side effects during module import;
+- return JSON-friendly `details` values;
+- document whether a rule is a warning or a hard error;
+- test against the supported FrameVitals version range;
+- avoid mutating the DataFrame passed into a check.
+
+FrameVitals gives every check an isolated DataFrame copy, but provider code should
+still behave as a pure predicate where practical.
+
+## Custom DatasetSource implementations
+
+A storage/query integration does not need to be added to FrameVitals core. The minimal
+protocol is deliberately small:
+
+```python
+from framevitals.sources import DatasetMetadata
+
+class MySource:
+    def inspect(self):
+        return DatasetMetadata(
+            name="warehouse.orders",
+            kind="remote",
+            format="my-engine",
+            rows=1_000_000,
+            columns=12,
+            size_bytes=None,
+            materialized=False,
+            supports_projection=False,
+            supports_streaming=False,
+        )
+
+    def load(self):
+        # Return the complete dataset as pandas only when an operation requires it.
+        return fetch_as_pandas()
+```
+
+Then pass the source directly:
+
+```python
+source = MySource()
+print(fv.inspect_source(source))
+report = fv.analyze(source, mode="quick")
+```
+
+Because the protocol is runtime-checkable, FrameVitals can consume compatible custom
+objects without maintaining a registry of every provider package.
+
+## Streaming source protocol
+
+To opt into source-aware batch execution, additionally implement `iter_batches`:
+
+```python
+class MyStreamingSource(MySource):
+    def iter_batches(self, *, batch_size=65_536, columns=None):
+        ...
+```
+
+A streaming adapter should preserve these invariants:
+
+1. `inspect()` reports the **true source shape**, not the retained sample shape.
+2. `iter_batches()` respects `batch_size` and requested projection where advertised.
+3. yielded batches are compatible with the Arrow-oriented streaming profiler.
+4. `load()` remains available for exact/full-row APIs.
+5. optional provider dependencies are imported lazily.
+6. the adapter does not pretend an exact operation is streaming merely because the
+   source supports batches.
+
+## Prefer standard interoperability over one-off adapters
+
+If a data library exposes the Arrow C Stream / PyCapsule interface, FrameVitals can
+usually consume it through the generic Arrow boundary instead of carrying a dedicated
+library dependency.
+
+That is how FrameVitals can interoperate with modern dataframe producers such as
+Polars while keeping Polars out of the runtime dependency graph.
+
+Use a dedicated adapter when the storage/query system provides capabilities that the
+generic Arrow boundary would lose. The DuckDB adapter is an example: it preserves a
+lazy relation, pushes column projection into DuckDB, obtains exact relation metadata,
+and streams Arrow batches without first constructing a complete in-memory table.
+
+## Execution provenance for extensions
+
+Extension authors should not invent incompatible exactness terminology. FrameVitals'
+shared execution schema uses fields such as:
+
+- `method`
+- `full_materialization`
+- `sampled`
+- `source_rows`
+- `source_columns`
+- `sample_rows`
+- `strategy`
+- `components`
+- `reason`
+
+See [Execution provenance](execution-provenance.md) for the current schema.

From b220168066b16daad524dd34788ae5a523956a61 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:58:32 +0530
Subject: [PATCH 327/439] examples: add third-party check plugin package

---
 examples/check_plugin/pyproject.toml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 examples/check_plugin/pyproject.toml

diff --git a/examples/check_plugin/pyproject.toml b/examples/check_plugin/pyproject.toml
new file mode 100644
index 0000000..a560913
--- /dev/null
+++ b/examples/check_plugin/pyproject.toml
@@ -0,0 +1,22 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "framevitals-example-checks"
+version = "0.1.0"
+description = "Minimal third-party FrameVitals check plugin example."
+requires-python = ">=3.11"
+dependencies = [
+    "framevitals>=0.1,<1",
+]
+
+[project.entry-points."framevitals.checks"]
+positive_revenue = "framevitals_example_checks:positive_revenue"
+preferred_plan = "framevitals_example_checks:preferred_plan"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]

From 1fec938a5b05e2c963015b62f9608ea629c7ed41 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:58:45 +0530
Subject: [PATCH 328/439] examples: implement third-party FrameVitals checks

---
 .../framevitals_example_checks/__init__.py    | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 examples/check_plugin/src/framevitals_example_checks/__init__.py

diff --git a/examples/check_plugin/src/framevitals_example_checks/__init__.py b/examples/check_plugin/src/framevitals_example_checks/__init__.py
new file mode 100644
index 0000000..e9928e0
--- /dev/null
+++ b/examples/check_plugin/src/framevitals_example_checks/__init__.py
@@ -0,0 +1,57 @@
+"""Example third-party checks discovered through FrameVitals entry points."""
+
+from __future__ import annotations
+
+import framevitals as fv
+
+
+@fv.check(
+    "positive revenue",
+    severity="error",
+    description="Revenue cannot be negative.",
+)
+def positive_revenue(dataframe):
+    if "revenue" not in dataframe.columns:
+        return {
+            "passed": False,
+            "message": "Required revenue column is missing.",
+            "details": {"required_column": "revenue"},
+        }
+
+    minimum = float(dataframe["revenue"].min())
+    return {
+        "passed": minimum >= 0,
+        "message": (
+            "Revenue is non-negative."
+            if minimum >= 0
+            else "Negative revenue records were found."
+        ),
+        "details": {"minimum": minimum},
+    }
+
+
+@fv.check(
+    "preferred plan domain",
+    severity="warning",
+    description="Plans should use the preferred product vocabulary.",
+)
+def preferred_plan(dataframe):
+    if "plan" not in dataframe.columns:
+        return {
+            "passed": False,
+            "message": "Plan column is missing.",
+            "details": {"required_column": "plan"},
+        }
+
+    allowed = {"basic", "pro", "enterprise"}
+    observed = set(dataframe["plan"].dropna().astype(str))
+    unexpected = sorted(observed - allowed)
+    return {
+        "passed": not unexpected,
+        "message": (
+            "Plan values use the preferred domain."
+            if not unexpected
+            else "Unexpected plan labels were found."
+        ),
+        "details": {"unexpected": unexpected},
+    }

From b837ce364c66eb892f65cdc5f328e161c3317788 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:58:56 +0530
Subject: [PATCH 329/439] examples: document check plugin template

---
 examples/check_plugin/README.md | 49 +++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 examples/check_plugin/README.md

diff --git a/examples/check_plugin/README.md b/examples/check_plugin/README.md
new file mode 100644
index 0000000..d4f4888
--- /dev/null
+++ b/examples/check_plugin/README.md
@@ -0,0 +1,49 @@
+# FrameVitals check plugin example
+
+This directory is a minimal installable third-party package that contributes custom
+checks to FrameVitals through standard Python entry points.
+
+It is intentionally separate from the main `framevitals` package so the example tests
+the same packaging/discovery boundary that an external project would use.
+
+## Install beside a FrameVitals checkout
+
+From the repository root:
+
+```bash
+python -m pip install -e .
+python -m pip install -e examples/check_plugin
+```
+
+Then discover the provider checks explicitly:
+
+```python
+import framevitals as fv
+
+checks = fv.discover_checks()
+for check in checks:
+    print(check.name, check.severity)
+```
+
+Use them directly or inside the quality gate:
+
+```python
+result = fv.gate(dataframe, custom_checks=checks)
+print(result.summary_text())
+```
+
+## Entry-point contract
+
+The example declares:
+
+```toml
+[project.entry-points."framevitals.checks"]
+positive_revenue = "framevitals_example_checks:positive_revenue"
+preferred_plan = "framevitals_example_checks:preferred_plan"
+```
+
+Provider packages may export `DataCheck` objects or compatible DataFrame callables.
+FrameVitals does **not** automatically import installed providers; applications opt in
+with `fv.discover_checks()` because loading entry points executes provider code.
+
+See `docs/extending-framevitals.md` for the extension-author guide.

From 5474e21d9fbddfd307fc613b9bc0d26b248d8c78 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 14:59:10 +0530
Subject: [PATCH 330/439] ci: verify external check plugin contract

---
 .github/workflows/plugin-example.yml | 87 ++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 .github/workflows/plugin-example.yml

diff --git a/.github/workflows/plugin-example.yml b/.github/workflows/plugin-example.yml
new file mode 100644
index 0000000..ba735bd
--- /dev/null
+++ b/.github/workflows/plugin-example.yml
@@ -0,0 +1,87 @@
+name: Check Plugin Example
+
+on:
+  push:
+    branches: [main, dev, "develop/august"]
+    paths:
+      - "src/framevitals/checks.py"
+      - "src/framevitals/plugins.py"
+      - "src/framevitals/operations.py"
+      - "examples/check_plugin/**"
+      - ".github/workflows/plugin-example.yml"
+  pull_request:
+    paths:
+      - "src/framevitals/checks.py"
+      - "src/framevitals/plugins.py"
+      - "src/framevitals/operations.py"
+      - "examples/check_plugin/**"
+      - ".github/workflows/plugin-example.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  smoke:
+    name: install and discover external checks
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: |
+            pyproject.toml
+            examples/check_plugin/pyproject.toml
+
+      - name: Install FrameVitals and example provider
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .
+          pip install --no-deps -e examples/check_plugin
+
+      - name: Verify dependency consistency
+        run: python -m pip check
+
+      - name: Discover and run provider checks
+        run: |
+          python - <<'PY'
+          import pandas as pd
+          import framevitals as fv
+
+          checks = fv.discover_checks()
+          assert [check.name for check in checks] == [
+              "positive revenue",
+              "preferred plan domain",
+          ]
+
+          passing = pd.DataFrame({
+              "revenue": [10.0, 20.0, 30.0],
+              "plan": ["basic", "pro", "enterprise"],
+          })
+          passing_result = fv.gate(passing, custom_checks=checks)
+          assert passing_result.status == "pass"
+
+          failing = pd.DataFrame({
+              "revenue": [10.0, -5.0, 30.0],
+              "plan": ["basic", "legacy", "enterprise"],
+          })
+          failing_result = fv.gate(failing, custom_checks=checks)
+          assert failing_result.status == "fail"
+          assert failing_result["checks"]["custom"]["summary"] == {
+              "checks": 2,
+              "passed": 0,
+              "warnings": 1,
+              "errors": 1,
+          }
+          PY

From cf5e4cd9cd17f71edc000fc1994b4b2ac644f2bc Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:04:03 +0530
Subject: [PATCH 331/439] test: cover snapshot monitoring CLI

---
 tests/test_cli_monitoring.py | 184 +++++++++++++++++++++++++++++++++++
 1 file changed, 184 insertions(+)
 create mode 100644 tests/test_cli_monitoring.py

diff --git a/tests/test_cli_monitoring.py b/tests/test_cli_monitoring.py
new file mode 100644
index 0000000..6f04a65
--- /dev/null
+++ b/tests/test_cli_monitoring.py
@@ -0,0 +1,184 @@
+import json
+
+from framevitals.cli import build_parser, main
+from framevitals.snapshots import load_snapshot
+
+
+def _dataset(path):
+    path.write_text(
+        "value,other,group\n"
+        "1,2,a\n"
+        "2,4,b\n"
+        "3,6,a\n"
+        "4,8,b\n"
+        "5,10,a\n"
+        "6,12,b\n",
+        encoding="utf-8",
+    )
+
+
+def test_snapshot_and_compare_snapshot_parsers():
+    parser = build_parser()
+
+    snapshot_args = parser.parse_args([
+        "snapshot",
+        "dataset.csv",
+        "--mode",
+        "quick",
+        "--workers",
+        "1",
+        "--output",
+        "snapshot.json",
+    ])
+    assert snapshot_args.command == "snapshot"
+    assert snapshot_args.file.name == "dataset.csv"
+    assert snapshot_args.mode == "quick"
+    assert snapshot_args.workers == 1
+    assert snapshot_args.output.name == "snapshot.json"
+
+    compare_args = parser.parse_args([
+        "compare-snapshots",
+        "baseline.json",
+        "current.json",
+        "--format",
+        "json",
+        "--fail-on-change",
+        "--output",
+        "diff.json",
+    ])
+    assert compare_args.command == "compare-snapshots"
+    assert compare_args.reference.name == "baseline.json"
+    assert compare_args.current.name == "current.json"
+    assert compare_args.format == "json"
+    assert compare_args.fail_on_change is True
+    assert compare_args.output.name == "diff.json"
+
+
+def test_snapshot_cli_writes_loadable_compact_state(tmp_path, monkeypatch, capsys):
+    dataset = tmp_path / "dataset.csv"
+    snapshot_path = tmp_path / "snapshot.json"
+    _dataset(dataset)
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "snapshot",
+            str(dataset),
+            "--mode",
+            "quick",
+            "--workers",
+            "1",
+            "--format",
+            "json",
+            "--output",
+            str(snapshot_path),
+        ],
+    )
+
+    assert main() == 0
+    stdout_payload = json.loads(capsys.readouterr().out)
+    saved = load_snapshot(snapshot_path)
+
+    assert stdout_payload["snapshot_schema_version"] == "1"
+    assert stdout_payload["fingerprint"] == saved["fingerprint"]
+    assert saved["source"]["filename"] == "dataset.csv"
+    assert saved["state"]["analysis_mode"] == "quick"
+    assert saved["state"]["config"]["artifacts"] is False
+    assert "profile" not in saved
+
+
+def test_compare_snapshots_cli_reports_unchanged_state(tmp_path, monkeypatch, capsys):
+    dataset = tmp_path / "dataset.csv"
+    snapshot_path = tmp_path / "snapshot.json"
+    diff_path = tmp_path / "diff.json"
+    _dataset(dataset)
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "snapshot",
+            str(dataset),
+            "--mode",
+            "quick",
+            "--workers",
+            "1",
+            "--format",
+            "json",
+            "--output",
+            str(snapshot_path),
+        ],
+    )
+    assert main() == 0
+    capsys.readouterr()
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "compare-snapshots",
+            str(snapshot_path),
+            str(snapshot_path),
+            "--format",
+            "json",
+            "--fail-on-change",
+            "--output",
+            str(diff_path),
+        ],
+    )
+    assert main() == 0
+    diff = json.loads(capsys.readouterr().out)
+
+    assert diff["changed"] is False
+    assert diff["schema"]["added_columns"] == []
+    assert diff["schema"]["removed_columns"] == []
+    assert json.loads(diff_path.read_text(encoding="utf-8")) == diff
+
+
+def test_compare_snapshots_cli_can_fail_ci_on_change(tmp_path, monkeypatch, capsys):
+    dataset = tmp_path / "dataset.csv"
+    baseline = tmp_path / "baseline.json"
+    current = tmp_path / "current.json"
+    _dataset(dataset)
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "snapshot",
+            str(dataset),
+            "--mode",
+            "quick",
+            "--workers",
+            "1",
+            "--output",
+            str(baseline),
+        ],
+    )
+    assert main() == 0
+    capsys.readouterr()
+
+    payload = json.loads(baseline.read_text(encoding="utf-8"))
+    payload["fingerprint"] = "0" * 64
+    payload["state"]["dataset"]["dtypes"]["new_column"] = "int64"
+    current.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "compare-snapshots",
+            str(baseline),
+            str(current),
+            "--format",
+            "terminal",
+            "--fail-on-change",
+        ],
+    )
+    assert main() == 1
+    rendered = capsys.readouterr().out
+
+    assert "FrameVitals snapshot diff" in rendered
+    assert "Changed         yes" in rendered
+    assert "Added columns   1" in rendered

From 54209609a3dc21411ed143a2fcd674c079e2a300 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:04:50 +0530
Subject: [PATCH 332/439] docs: add snapshot monitoring workflow

---
 docs/monitoring.md | 120 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 120 insertions(+)
 create mode 100644 docs/monitoring.md

diff --git a/docs/monitoring.md b/docs/monitoring.md
new file mode 100644
index 0000000..f9f45a7
--- /dev/null
+++ b/docs/monitoring.md
@@ -0,0 +1,120 @@
+# Monitoring with snapshots
+
+FrameVitals snapshots turn a full analysis result into compact, versioned monitoring
+state. They are designed for workflows where you want to detect meaningful changes
+without storing every raw production batch.
+
+## Create a snapshot in Python
+
+```python
+import framevitals as fv
+
+report = fv.analyze("production.parquet", mode="quick")
+snapshot = report.snapshot("production.snapshot.json")
+```
+
+A snapshot retains compact state such as:
+
+- source identity;
+- schema and dtypes;
+- missingness;
+- duplicate rate;
+- health score;
+- ML-readiness score;
+- finding codes;
+- analysis configuration;
+- a deterministic state fingerprint.
+
+It does **not** embed the full raw dataset.
+
+## Snapshot from the CLI
+
+```bash
+framevitals snapshot production.parquet \
+  --mode quick \
+  --output production.snapshot.json
+```
+
+Snapshot generation always disables filesystem analysis artifacts. The requested
+snapshot file is the only explicit monitoring artifact written by the command.
+
+Use JSON on stdout when integrating with another tool:
+
+```bash
+framevitals snapshot production.parquet \
+  --mode quick \
+  --format json \
+  --output production.snapshot.json
+```
+
+## Compare two snapshots
+
+```bash
+framevitals compare-snapshots \
+  baseline.snapshot.json \
+  production.snapshot.json
+```
+
+The comparison reports:
+
+- whether the state fingerprint changed;
+- added or removed columns;
+- dtype changes;
+- per-column missingness changes;
+- health-score delta;
+- ML-readiness delta;
+- new findings;
+- resolved findings.
+
+For machine-readable output:
+
+```bash
+framevitals compare-snapshots \
+  baseline.snapshot.json \
+  production.snapshot.json \
+  --format json \
+  --output snapshot-diff.json
+```
+
+## Fail CI when state changes
+
+Snapshot change is not automatically treated as failure because many changes are
+legitimate. When a pipeline explicitly wants fingerprint stability, opt in:
+
+```bash
+framevitals compare-snapshots \
+  baseline.snapshot.json \
+  production.snapshot.json \
+  --fail-on-change
+```
+
+The command returns exit code `1` when the fingerprints differ and `0` when they are
+equal.
+
+For data-quality enforcement, prefer `framevitals gate` when you want semantic
+contract/drift thresholds rather than a strict "anything changed" rule.
+
+## Persistent history in Python
+
+For repeated local monitoring, `SnapshotHistory` stores compact snapshots under one
+directory:
+
+```python
+history = fv.SnapshotHistory(".framevitals/history")
+history.add(report.snapshot(), label="production")
+
+latest = history.latest()
+previous = history.previous()
+change = history.compare_latest()
+```
+
+The default `.framevitals/` runtime directory is ignored by Git.
+
+## Snapshot compatibility
+
+Snapshots carry their own schema version. `fv.load_snapshot()` validates that version
+before returning an `AnalysisSnapshot`, so incompatible monitoring state fails loudly
+instead of being compared with unknown semantics.
+
+During the `0.x` series, snapshot/result schemas may still evolve. Breaking changes
+must be recorded in the changelog and release notes before promotion to `main`.

From 9c60f13b5a2b477c7b7582215a5a362be5c15984 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:05:28 +0530
Subject: [PATCH 333/439] perf: define initial profiling guardrails

---
 benchmarks/performance_budgets.json | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 benchmarks/performance_budgets.json

diff --git a/benchmarks/performance_budgets.json b/benchmarks/performance_budgets.json
new file mode 100644
index 0000000..505bb40
--- /dev/null
+++ b/benchmarks/performance_budgets.json
@@ -0,0 +1,14 @@
+{
+  "schema_version": "1",
+  "notes": "Initial ceilings are intentionally generous. Tighten only after collecting stable hosted-run history.",
+  "scenarios": {
+    "core_profile_150k_x12": {
+      "max_seconds": 25.0,
+      "max_peak_rss_mb": 1200.0
+    },
+    "parquet_profile_150k_x12": {
+      "max_seconds": 30.0,
+      "max_peak_rss_mb": 1400.0
+    }
+  }
+}

From f000c4a5f7902be4079925971330759c1b9f41cd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:05:48 +0530
Subject: [PATCH 334/439] perf: add deterministic performance guardrail

---
 benchmarks/performance_guardrail.py | 198 ++++++++++++++++++++++++++++
 1 file changed, 198 insertions(+)
 create mode 100644 benchmarks/performance_guardrail.py

diff --git a/benchmarks/performance_guardrail.py b/benchmarks/performance_guardrail.py
new file mode 100644
index 0000000..e81d5ed
--- /dev/null
+++ b/benchmarks/performance_guardrail.py
@@ -0,0 +1,198 @@
+"""Deterministic catastrophic-regression guardrails for FrameVitals profiling.
+
+These checks are deliberately broader than microbenchmarks. Hosted CI hardware is
+noisy, so the initial budgets only fail large time/RSS regressions. Raw measurements
+are written as JSON so the ceilings can be tightened later from observed history.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import resource
+import tempfile
+import time
+from pathlib import Path
+from typing import Any, Callable
+
+import numpy as np
+import pandas as pd
+
+import framevitals as fv
+
+
+ROWS = 150_000
+NUMERIC_COLUMNS = 8
+CATEGORICAL_COLUMNS = 4
+
+
+def _peak_rss_mb() -> float:
+    usage = float(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
+    # Linux reports KiB; macOS reports bytes. CI is Linux, but keep the script
+    # useful locally as well.
+    if os.uname().sysname.lower() == "darwin":
+        return usage / (1024.0 * 1024.0)
+    return usage / 1024.0
+
+
+def _frame(rows: int = ROWS) -> pd.DataFrame:
+    rng = np.random.default_rng(42)
+    payload: dict[str, Any] = {}
+    for index in range(NUMERIC_COLUMNS):
+        values = rng.normal(loc=index, scale=1.0 + index / 10.0, size=rows)
+        values[::997] = np.nan
+        payload[f"value_{index}"] = values
+    for index in range(CATEGORICAL_COLUMNS):
+        payload[f"group_{index}"] = np.take(
+            np.array([f"g{index}-{item}" for item in range(12)], dtype=object),
+            np.arange(rows) % 12,
+        )
+    return pd.DataFrame(payload)
+
+
+def _measure(name: str, operation: Callable[[], dict[str, Any]]) -> dict[str, Any]:
+    before_rss = _peak_rss_mb()
+    started = time.perf_counter()
+    result = operation()
+    elapsed = time.perf_counter() - started
+    after_rss = _peak_rss_mb()
+    return {
+        "name": name,
+        "elapsed_seconds": round(float(elapsed), 6),
+        "peak_rss_mb": round(float(after_rss), 3),
+        "rss_growth_mb": round(max(0.0, float(after_rss - before_rss)), 3),
+        "result": result,
+    }
+
+
+def _core_profile() -> dict[str, Any]:
+    frame = _frame()
+    profile = fv.profile(frame)
+    assert profile["shape"] == {"rows": ROWS, "columns": NUMERIC_COLUMNS + CATEGORICAL_COLUMNS}
+    return {
+        "rows": ROWS,
+        "columns": NUMERIC_COLUMNS + CATEGORICAL_COLUMNS,
+        "diagnostic": profile.diagnostic,
+    }
+
+
+def _parquet_profile() -> dict[str, Any]:
+    try:
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+    except ImportError as exc:  # pragma: no cover - workflow installs arrow
+        raise RuntimeError(
+            'Parquet guardrail requires: pip install "framevitals[arrow]"'
+        ) from exc
+
+    frame = _frame()
+    with tempfile.TemporaryDirectory(prefix="framevitals-perf-") as directory:
+        path = Path(directory) / "guardrail.parquet"
+        pq.write_table(
+            pa.Table.from_pandas(frame, preserve_index=False),
+            path,
+            row_group_size=8_192,
+        )
+        del frame
+        profile = fv.profile(path)
+
+    assert profile["shape"] == {"rows": ROWS, "columns": NUMERIC_COLUMNS + CATEGORICAL_COLUMNS}
+    streaming = profile.get("streaming_metadata", {})
+    assert streaming.get("enabled") is True
+    assert streaming.get("full_materialization") is False
+    return {
+        "rows": ROWS,
+        "columns": NUMERIC_COLUMNS + CATEGORICAL_COLUMNS,
+        "diagnostic": profile.diagnostic,
+        "streaming": True,
+    }
+
+
+SCENARIOS: dict[str, Callable[[], dict[str, Any]]] = {
+    "core_profile_150k_x12": _core_profile,
+    "parquet_profile_150k_x12": _parquet_profile,
+}
+
+
+def _load_budgets(path: Path) -> dict[str, dict[str, float]]:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    if payload.get("schema_version") != "1":
+        raise ValueError("Unsupported performance budget schema version.")
+    scenarios = payload.get("scenarios")
+    if not isinstance(scenarios, dict):
+        raise ValueError("Performance budget file is missing scenarios.")
+    return scenarios
+
+
+def run_guardrails(budget_path: Path) -> dict[str, Any]:
+    budgets = _load_budgets(budget_path)
+    measurements: list[dict[str, Any]] = []
+    failures: list[dict[str, Any]] = []
+
+    for name, operation in SCENARIOS.items():
+        if name not in budgets:
+            raise ValueError(f"Missing performance budget for scenario: {name}")
+        budget = budgets[name]
+        measurement = _measure(name, operation)
+        max_seconds = float(budget["max_seconds"])
+        max_peak_rss_mb = float(budget["max_peak_rss_mb"])
+        measurement["budget"] = {
+            "max_seconds": max_seconds,
+            "max_peak_rss_mb": max_peak_rss_mb,
+        }
+        measurement["passed"] = (
+            measurement["elapsed_seconds"] <= max_seconds
+            and measurement["peak_rss_mb"] <= max_peak_rss_mb
+        )
+        measurements.append(measurement)
+
+        if measurement["elapsed_seconds"] > max_seconds:
+            failures.append({
+                "scenario": name,
+                "metric": "elapsed_seconds",
+                "observed": measurement["elapsed_seconds"],
+                "budget": max_seconds,
+            })
+        if measurement["peak_rss_mb"] > max_peak_rss_mb:
+            failures.append({
+                "scenario": name,
+                "metric": "peak_rss_mb",
+                "observed": measurement["peak_rss_mb"],
+                "budget": max_peak_rss_mb,
+            })
+
+    return {
+        "schema_version": "1",
+        "rows": ROWS,
+        "scenario_count": len(measurements),
+        "passed": not failures,
+        "measurements": measurements,
+        "failures": failures,
+    }
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--budgets",
+        type=Path,
+        default=Path(__file__).with_name("performance_budgets.json"),
+    )
+    parser.add_argument("--output", type=Path, default=None)
+    return parser
+
+
+def main() -> int:
+    args = _parser().parse_args()
+    result = run_guardrails(args.budgets)
+    rendered = json.dumps(result, indent=2, default=str)
+    if args.output is not None:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(rendered + "\n", encoding="utf-8")
+    print(rendered)
+    return 0 if result["passed"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

From 6ff54aa5ab69f7c4393aabef6e507b1f76126d1c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:06:04 +0530
Subject: [PATCH 335/439] ci: enforce catastrophic performance guardrails

---
 .github/workflows/performance-guardrail.yml | 101 ++++++++++++++++++++
 1 file changed, 101 insertions(+)
 create mode 100644 .github/workflows/performance-guardrail.yml

diff --git a/.github/workflows/performance-guardrail.yml b/.github/workflows/performance-guardrail.yml
new file mode 100644
index 0000000..2e00bf4
--- /dev/null
+++ b/.github/workflows/performance-guardrail.yml
@@ -0,0 +1,101 @@
+name: Performance Guardrail
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "src/framevitals/profiler.py"
+      - "src/framevitals/streaming_profile.py"
+      - "src/framevitals/sources.py"
+      - "src/framevitals/focused.py"
+      - "src/framevitals/native_core.py"
+      - "rust/**"
+      - "benchmarks/performance_guardrail.py"
+      - "benchmarks/performance_budgets.json"
+      - ".github/workflows/performance-guardrail.yml"
+  push:
+    branches: [main, dev, "develop/august"]
+    paths:
+      - "src/framevitals/profiler.py"
+      - "src/framevitals/streaming_profile.py"
+      - "src/framevitals/sources.py"
+      - "src/framevitals/focused.py"
+      - "src/framevitals/native_core.py"
+      - "rust/**"
+      - "benchmarks/performance_guardrail.py"
+      - "benchmarks/performance_budgets.json"
+      - ".github/workflows/performance-guardrail.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  profile-guardrails:
+    name: profiling time + RSS ceilings
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install profiling capabilities
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[arrow]"
+
+      - name: Verify dependency consistency
+        run: python -m pip check
+
+      - name: Run performance guardrails
+        env:
+          FRAMEVITALS_BACKEND: numpy
+        run: |
+          python benchmarks/performance_guardrail.py \
+            --output performance-guardrail.json
+
+      - name: Publish job summary
+        if: always()
+        run: |
+          if [[ ! -f performance-guardrail.json ]]; then
+            exit 0
+          fi
+          python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
+          import json
+          from pathlib import Path
+
+          payload = json.loads(Path("performance-guardrail.json").read_text())
+          print("## FrameVitals performance guardrail")
+          print()
+          print("| Scenario | Time (s) | Time budget | Peak RSS (MB) | RSS budget | Result |")
+          print("| --- | ---: | ---: | ---: | ---: | --- |")
+          for item in payload["measurements"]:
+              budget = item["budget"]
+              result = "PASS" if item["passed"] else "FAIL"
+              print(
+                  f"| {item['name']} | {item['elapsed_seconds']:.3f} | "
+                  f"{budget['max_seconds']:.1f} | {item['peak_rss_mb']:.1f} | "
+                  f"{budget['max_peak_rss_mb']:.1f} | {result} |"
+              )
+          PY
+
+      - name: Upload guardrail measurements
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: performance-guardrail
+          path: performance-guardrail.json
+          if-no-files-found: ignore
+          retention-days: 30

From b03a9a3551aed8cc15b614865e103573d137b11e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:07:41 +0530
Subject: [PATCH 336/439] dev: add lightweight pre-commit guardrails

---
 .pre-commit-config.yaml | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 .pre-commit-config.yaml

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000..4b87833
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,35 @@
+minimum_pre_commit_version: "4.0.0"
+
+default_install_hook_types:
+  - pre-commit
+  - pre-push
+
+repos:
+  - repo: local
+    hooks:
+      - id: framevitals-critical-ruff
+        name: FrameVitals critical Python lint
+        entry: python -m ruff check --select E9,F821,F822,F823 src/framevitals tests
+        language: system
+        pass_filenames: false
+        files: ^(src/framevitals|tests)/.*\.py$
+
+      - id: framevitals-compile
+        name: FrameVitals Python compile check
+        entry: python -m compileall -q src/framevitals
+        language: system
+        pass_filenames: false
+        files: ^src/framevitals/.*\.py$
+
+      - id: framevitals-public-contracts
+        name: FrameVitals public contract tests
+        entry: >-
+          python -m pytest -q
+          tests/test_public_surface_contract.py
+          tests/test_provenance.py
+          tests/test_source_inspection.py
+          tests/test_check_plugins.py
+        language: system
+        pass_filenames: false
+        always_run: true
+        stages: [pre-push]

From 3cd2bca2587acbed2cf012d5861be5ac66b1d654 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:08:00 +0530
Subject: [PATCH 337/439] ci: validate contributor pre-commit hooks

---
 .github/workflows/developer-guardrails.yml | 57 ++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 .github/workflows/developer-guardrails.yml

diff --git a/.github/workflows/developer-guardrails.yml b/.github/workflows/developer-guardrails.yml
new file mode 100644
index 0000000..7b4eecd
--- /dev/null
+++ b/.github/workflows/developer-guardrails.yml
@@ -0,0 +1,57 @@
+name: Developer Guardrails
+
+on:
+  pull_request:
+    paths:
+      - ".pre-commit-config.yaml"
+      - "pyproject.toml"
+      - "src/framevitals/**"
+      - "tests/**"
+      - ".github/workflows/developer-guardrails.yml"
+  push:
+    branches: [main, dev, "develop/august"]
+    paths:
+      - ".pre-commit-config.yaml"
+      - "pyproject.toml"
+      - "src/framevitals/**"
+      - "tests/**"
+      - ".github/workflows/developer-guardrails.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  hooks:
+    name: pre-commit + pre-push contracts
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install development environment
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Verify dependency consistency
+        run: python -m pip check
+
+      - name: Run commit-stage hooks
+        run: pre-commit run --all-files --hook-stage pre-commit --show-diff-on-failure
+
+      - name: Run pre-push public contract hooks
+        run: pre-commit run --all-files --hook-stage pre-push --show-diff-on-failure

From 804f5a1a282659c91c724f36438dea38253b0bc9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:09:34 +0530
Subject: [PATCH 338/439] test: cover system capability CLI

---
 tests/test_cli_system_info.py | 85 +++++++++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 tests/test_cli_system_info.py

diff --git a/tests/test_cli_system_info.py b/tests/test_cli_system_info.py
new file mode 100644
index 0000000..1a78a5b
--- /dev/null
+++ b/tests/test_cli_system_info.py
@@ -0,0 +1,85 @@
+import json
+
+from framevitals.cli import build_parser, main
+
+
+def test_system_info_parser_exposes_gpu_and_output_controls():
+    parser = build_parser()
+
+    args = parser.parse_args([
+        "system-info",
+        "--no-probe-gpu",
+        "--format",
+        "json",
+        "--output",
+        "system.json",
+    ])
+
+    assert args.command == "system-info"
+    assert args.probe_gpu is False
+    assert args.format == "json"
+    assert args.output.name == "system.json"
+
+
+def test_system_info_cli_routes_to_canonical_capability_api(
+    tmp_path,
+    monkeypatch,
+    capsys,
+):
+    output = tmp_path / "system.json"
+    expected = {
+        "python": "3.12.0",
+        "backend": "numpy",
+        "native": {
+            "available": False,
+            "reason": "test fixture",
+        },
+        "gpu": {
+            "probed": False,
+            "available": False,
+        },
+    }
+    calls = []
+
+    def fake_system_info(*, probe_gpu=True):
+        calls.append(probe_gpu)
+        return expected
+
+    monkeypatch.setattr("framevitals.acceleration.system_info", fake_system_info)
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "framevitals",
+            "system-info",
+            "--no-probe-gpu",
+            "--format",
+            "json",
+            "--output",
+            str(output),
+        ],
+    )
+
+    assert main() == 0
+    assert calls == [False]
+    stdout = json.loads(capsys.readouterr().out)
+    saved = json.loads(output.read_text(encoding="utf-8"))
+    assert stdout == expected
+    assert saved == expected
+
+
+def test_system_info_terminal_render_is_human_readable(monkeypatch, capsys):
+    monkeypatch.setattr(
+        "framevitals.acceleration.system_info",
+        lambda *, probe_gpu=True: {
+            "backend": "numpy",
+            "native": {"available": True},
+        },
+    )
+    monkeypatch.setattr("sys.argv", ["framevitals", "system-info"])
+
+    assert main() == 0
+    rendered = capsys.readouterr().out
+    assert "FrameVitals system info" in rendered
+    assert "Backend" in rendered
+    assert "Native" in rendered
+    assert "available: True" in rendered

From 55022a26152599bbfd349cf187fda5eef60d2149 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:10:01 +0530
Subject: [PATCH 339/439] docs: add reproducible support guidance

---
 SUPPORT.md | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100644 SUPPORT.md

diff --git a/SUPPORT.md b/SUPPORT.md
new file mode 100644
index 0000000..4995e30
--- /dev/null
+++ b/SUPPORT.md
@@ -0,0 +1,97 @@
+# FrameVitals support
+
+FrameVitals is alpha software and the fastest way to fix a problem is a small,
+reproducible report that shows **what input/source path was used, which optional
+capabilities were installed, and what FrameVitals actually executed**.
+
+## Before opening an issue
+
+1. Reproduce the problem on the newest compatible FrameVitals release or the relevant
+   development branch.
+2. Reduce the input to the smallest dataset that still reproduces the behavior.
+3. Check whether the same call behaves differently with a pandas DataFrame versus the
+   original source adapter.
+4. Include execution/source metadata when the problem involves memory, sampling,
+   streaming, or performance.
+
+Useful commands:
+
+```bash
+framevitals --version
+framevitals system-info --no-probe-gpu --format json
+framevitals inspect path/to/dataset.parquet --format json
+```
+
+If the problem specifically involves CUDA/GPU probing, run `framevitals system-info`
+without `--no-probe-gpu` as well.
+
+## What to include in a bug report
+
+Please provide:
+
+- FrameVitals version or commit SHA;
+- Python version and operating system;
+- the exact command or Python call;
+- the smallest reproducible input shape/schema;
+- which optional extras are installed (`arrow`, `duckdb`, `excel`, `plot`, `ml`, `ai`,
+  or `web`);
+- `framevitals system-info --no-probe-gpu --format json` output when relevant;
+- `framevitals inspect ... --format json` output for source/streaming issues;
+- the complete exception traceback;
+- expected behavior and actual behavior.
+
+For a focused diagnostic, include its execution metadata when possible:
+
+```python
+result = fv.statistics(data, mode="quick")
+print(result.execution)
+```
+
+This tells maintainers whether the operation sampled, streamed, or fully materialized
+its source.
+
+## Reproduction datasets
+
+Do **not** upload confidential, personal, regulated, proprietary, or customer data to a
+public issue. Prefer one of these:
+
+- a tiny synthetic DataFrame;
+- a short CSV created specifically for the reproduction;
+- code that deterministically generates the failing data pattern;
+- schema/source metadata with sensitive values removed.
+
+## Performance reports
+
+For performance regressions, include:
+
+- dataset rows/columns and approximate file size;
+- source format and whether `inspect_source()` reports streaming/projection support;
+- analysis mode;
+- `result.execution` when available;
+- wall-clock time and peak memory if measured;
+- whether `FRAMEVITALS_BACKEND` was configured explicitly;
+- native-core availability from `system-info`.
+
+The repository also contains reproducible benchmark and catastrophic-regression
+workflows under `benchmarks/` and `.github/workflows/`.
+
+## Feature requests
+
+A strong feature request explains the workflow problem before proposing a new module.
+FrameVitals deliberately prefers a small analyze → compare → validate → gate → monitor
+surface over accumulating unrelated analytics features.
+
+For source integrations, first check whether the producer already supports the Arrow C
+Stream / PyCapsule protocol; a standard interoperability boundary is preferable to a
+library-specific dependency when it preserves the required semantics.
+
+## Security issues
+
+Do not report vulnerabilities, credential exposure, or sensitive-data security issues
+through a normal public support thread. Follow the repository's `SECURITY.md` process.
+
+## General questions
+
+Questions that can become durable documentation are welcome. When possible, include a
+small code example and the desired outcome so the answer can be turned into a docs or
+example improvement for future users.

From 2509ce18e962012b496d37929fb7f287e5384e92 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:10:36 +0530
Subject: [PATCH 340/439] ci: test declared minimum core dependencies

---
 .github/workflows/minimum-dependencies.yml | 86 ++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 .github/workflows/minimum-dependencies.yml

diff --git a/.github/workflows/minimum-dependencies.yml b/.github/workflows/minimum-dependencies.yml
new file mode 100644
index 0000000..723a871
--- /dev/null
+++ b/.github/workflows/minimum-dependencies.yml
@@ -0,0 +1,86 @@
+name: Minimum Dependencies
+
+on:
+  pull_request:
+    paths:
+      - "pyproject.toml"
+      - "src/framevitals/**"
+      - "tests/**"
+      - ".github/workflows/minimum-dependencies.yml"
+  push:
+    branches: [main, dev, "develop/august"]
+    paths:
+      - "pyproject.toml"
+      - "src/framevitals/**"
+      - "tests/**"
+      - ".github/workflows/minimum-dependencies.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lower-bounds:
+    name: Python 3.11 lower bounds
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install declared minimum core dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install \
+            "numpy==1.26.0" \
+            "pandas==2.2.0" \
+            "scipy==1.13.0" \
+            "statsmodels==0.14.0" \
+            "scikit-learn==1.5.0" \
+            "pytest>=8.2" \
+            "hypothesis>=6.100"
+          pip install --no-deps -e .
+
+      - name: Verify dependency consistency
+        run: python -m pip check
+
+      - name: Verify installed lower bounds
+        run: |
+          python - <<'PY'
+          import numpy
+          import pandas
+          import scipy
+          import sklearn
+          import statsmodels
+
+          expected = {
+              "numpy": "1.26.0",
+              "pandas": "2.2.0",
+              "scipy": "1.13.0",
+              "sklearn": "1.5.0",
+              "statsmodels": "0.14.0",
+          }
+          observed = {
+              "numpy": numpy.__version__,
+              "pandas": pandas.__version__,
+              "scipy": scipy.__version__,
+              "sklearn": sklearn.__version__,
+              "statsmodels": statsmodels.__version__,
+          }
+          assert observed == expected, (observed, expected)
+          PY
+
+      - name: Run core test suite at lower bounds
+        run: pytest -q

From f1b70013c6a35a03277d99e11c78ec6aee6c9a9f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:11:01 +0530
Subject: [PATCH 341/439] ci: add Windows and macOS package smoke tests

---
 .github/workflows/platform-smoke.yml | 84 ++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 .github/workflows/platform-smoke.yml

diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml
new file mode 100644
index 0000000..b750b24
--- /dev/null
+++ b/.github/workflows/platform-smoke.yml
@@ -0,0 +1,84 @@
+name: Platform Smoke
+
+on:
+  pull_request:
+    paths:
+      - "pyproject.toml"
+      - "src/framevitals/**"
+      - "tests/test_public_api.py"
+      - "tests/test_public_surface_contract.py"
+      - "tests/test_source_inspection.py"
+      - "tests/test_diagnostic_results.py"
+      - "tests/test_snapshots.py"
+      - "tests/test_cli.py"
+      - "tests/test_cli_monitoring.py"
+      - "tests/test_cli_system_info.py"
+      - ".github/workflows/platform-smoke.yml"
+  push:
+    branches: [main, dev, "develop/august"]
+    paths:
+      - "pyproject.toml"
+      - "src/framevitals/**"
+      - "tests/test_public_api.py"
+      - "tests/test_public_surface_contract.py"
+      - "tests/test_source_inspection.py"
+      - "tests/test_diagnostic_results.py"
+      - "tests/test_snapshots.py"
+      - "tests/test_cli.py"
+      - "tests/test_cli_monitoring.py"
+      - "tests/test_cli_system_info.py"
+      - ".github/workflows/platform-smoke.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  smoke:
+    name: ${{ matrix.os }} / Python 3.12
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install core package and test tools
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+
+      - name: Verify dependency consistency
+        run: python -m pip check
+
+      - name: Verify CLI install
+        run: |
+          framevitals --version
+          framevitals --help
+
+      - name: Run cross-platform public smoke suite
+        run: >-
+          python -m pytest -q
+          tests/test_public_api.py
+          tests/test_public_surface_contract.py
+          tests/test_source_inspection.py
+          tests/test_diagnostic_results.py
+          tests/test_snapshots.py
+          tests/test_cli.py
+          tests/test_cli_monitoring.py
+          tests/test_cli_system_info.py

From 95ed294fc3ff1fe9360fa79ea3da13bc19ad6785 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:11:42 +0530
Subject: [PATCH 342/439] security: add CodeQL analysis

---
 .github/workflows/codeql.yml | 45 ++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 .github/workflows/codeql.yml

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
new file mode 100644
index 0000000..26001ff
--- /dev/null
+++ b/.github/workflows/codeql.yml
@@ -0,0 +1,45 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [main, dev]
+  pull_request:
+    branches: [main, dev]
+  schedule:
+    - cron: "17 4 * * 1"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  analyze:
+    name: ${{ matrix.language }} security analysis
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      packages: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        language:
+          - python
+          - javascript-typescript
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v4
+        with:
+          languages: ${{ matrix.language }}
+
+      - name: Analyze
+        uses: github/codeql-action/analyze@v4
+        with:
+          category: "/language:${{ matrix.language }}"

From 7fc696fc760a230649ee2a1813b4d3cbda19493c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 15:12:28 +0530
Subject: [PATCH 343/439] docs: define compatibility and stability policy

---
 docs/stability.md | 138 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 138 insertions(+)
 create mode 100644 docs/stability.md

diff --git a/docs/stability.md b/docs/stability.md
new file mode 100644
index 0000000..f0b9149
--- /dev/null
+++ b/docs/stability.md
@@ -0,0 +1,138 @@
+# Stability and compatibility policy
+
+FrameVitals follows semantic versioning, but the project is currently in the `0.x`
+series. The purpose of this policy is to make that maturity level explicit without
+using "alpha" as an excuse for arbitrary breakage.
+
+## Python versions
+
+The current package metadata supports:
+
+- Python 3.11
+- Python 3.12
+- Python 3.13
+
+The main test matrix exercises all three. Separate smoke workflows exercise public
+package/CLI behavior on Windows and macOS in addition to the primary Linux CI.
+
+## Dependency lower bounds
+
+Minimum core dependency versions declared in `pyproject.toml` are tested in a dedicated
+Python 3.11 lower-bound workflow. If FrameVitals begins relying on behavior unavailable
+at an advertised lower bound, the project must either restore compatibility or raise
+the declared minimum deliberately.
+
+Optional capability groups (`arrow`, `duckdb`, `excel`, `plot`, `ml`, `ai`, `web`,
+`docs`) remain independently installable. Core import/CLI behavior should not require
+an optional dependency merely because the corresponding module exists in the source
+tree.
+
+## Public Python surface
+
+The package-root workflow APIs are the primary public surface:
+
+- `inspect_source`
+- `analyze` / `plan`
+- focused diagnostics such as `profile`, `health`, `statistics`, and `relationships`
+- `compare`
+- `infer_contract` / `validate`
+- `check` / `run_checks` / `discover_checks`
+- `gate`
+- snapshot/history helpers
+
+A regression test locks the exported package surface so accidental symbol removal is
+caught in CI.
+
+Modules and helpers not exported at package root may still be intentionally reusable,
+but should not be assumed to have the same compatibility promise unless documented.
+
+## Dict-compatible result objects
+
+During `0.x`, public result objects remain subclasses of `dict` to preserve existing
+mapping and JSON-oriented callers while adding helper methods.
+
+FrameVitals avoids injecting Python-only wrapper metadata into serialized payloads.
+For example, `DiagnosticResult.diagnostic` is object metadata and does not become a
+new JSON key merely because a focused result is wrapped.
+
+## Versioned machine-readable schemas
+
+Several persisted/automation-facing structures carry explicit schema versions:
+
+- analysis results: `result_schema_version`
+- monitoring snapshots: `snapshot_schema_version`
+- execution provenance: `execution_schema_version`
+- data contracts: contract `version`
+
+A schema-version bump should accompany an intentionally incompatible machine-readable
+change. New additive fields do not necessarily require a bump when older consumers
+can ignore them safely.
+
+## Execution semantics
+
+Source-aware execution is part of the public behavior, not an implementation detail.
+When a result exposes execution provenance, FrameVitals should not silently relabel:
+
+- sampled work as exact;
+- a sample shape as source shape;
+- full pandas materialization as streaming;
+- approximate cardinality/duplicate estimates as exact values.
+
+Performance optimizations, including the optional native Rust core, must preserve
+public result semantics. Acceleration is not allowed to create a separate correctness
+contract.
+
+## Extension compatibility
+
+Third-party check plugins use the `framevitals.checks` Python entry-point group.
+Plugin discovery is explicit and never automatic because loading an entry point
+executes provider code.
+
+Custom data sources integrate through the `DatasetSource` / `StreamingDatasetSource`
+protocols. Prefer standard Arrow interoperability over a library-specific dependency
+when the protocol preserves the required metadata and execution semantics.
+
+## What may change during 0.x
+
+Before `1.0`, minor releases may still revise:
+
+- scoring thresholds;
+- finding codes or wording;
+- result structure not protected by an explicit schema contract;
+- source-adapter implementation details;
+- experimental analysis modules;
+- extension points that are explicitly marked provisional.
+
+Breaking changes should be documented in `CHANGELOG.md` and release notes before they
+reach `main`. Avoid breaking package-root workflow signatures unless the improvement
+is substantial enough to justify migration cost.
+
+## Deprecation direction
+
+During `0.x`, FrameVitals may occasionally make direct breaking changes when maintaining
+two behaviors would create more ambiguity than value. Where practical, prefer a
+warning/migration period.
+
+After `1.0`, the intended policy is stricter: public package-root APIs and versioned
+schemas should use documented deprecation periods before removal, except when a change
+is required to address a security or correctness issue.
+
+## CI as the compatibility contract
+
+The repository intentionally separates compatibility concerns into dedicated lanes:
+
+- Python 3.11/3.12/3.13 core tests;
+- declared minimum dependency tests;
+- Arrow streaming tests;
+- DuckDB interoperability tests;
+- Polars-through-Arrow protocol tests;
+- Windows/macOS public smoke tests;
+- native Rust bridge checks;
+- plugin-provider install/discovery tests;
+- package/wheel validation;
+- strict documentation build;
+- performance guardrails;
+- CodeQL security analysis.
+
+A documented compatibility claim should ideally have a corresponding automated lane
+before it is treated as a project promise.

From 7e8b5691b00d30cac59bdacf9d8b3d1a06a94908 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:16:03 +0530
Subject: [PATCH 344/439] chore: prepare stabilization patch

---
 tmp-stabilization-placeholder | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 tmp-stabilization-placeholder

diff --git a/tmp-stabilization-placeholder b/tmp-stabilization-placeholder
new file mode 100644
index 0000000..b3a4252
--- /dev/null
+++ b/tmp-stabilization-placeholder
@@ -0,0 +1 @@
+placeholder
\ No newline at end of file

From 50e17345d247bd1f777b86ae971ee1ccd171b4a8 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:16:14 +0530
Subject: [PATCH 345/439] chore: remove temporary stabilization placeholder

---
 tmp-stabilization-placeholder | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 tmp-stabilization-placeholder

diff --git a/tmp-stabilization-placeholder b/tmp-stabilization-placeholder
deleted file mode 100644
index b3a4252..0000000
--- a/tmp-stabilization-placeholder
+++ /dev/null
@@ -1 +0,0 @@
-placeholder
\ No newline at end of file

From 38de281c472d05c7c84655396624e215866f041c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:18:06 +0530
Subject: [PATCH 346/439] fix: include pre-commit in development extra

---
 pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyproject.toml b/pyproject.toml
index 438b659..4404f5d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -100,6 +100,7 @@ dev = [
     "build>=1.2",
     "twine>=5.1",
     "ruff>=0.6",
+    "pre-commit>=4.0",
     "openpyxl>=3.1",
     "xlrd>=2.0",
     "matplotlib>=3.8",

From f678e32726c116a952711080022e36bee3b06d4f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:18:42 +0530
Subject: [PATCH 347/439] fix: normalize Arrow view types for streaming compute

---
 src/framevitals/sources.py | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/src/framevitals/sources.py b/src/framevitals/sources.py
index ee146bf..0bad2b3 100644
--- a/src/framevitals/sources.py
+++ b/src/framevitals/sources.py
@@ -82,6 +82,39 @@ def load(self) -> pd.DataFrame:
         return self.dataframe.copy()
 
 
+def _normalize_arrow_table_types(table: Any) -> Any:
+    """Cast Arrow view types to compute-compatible equivalents without pandas."""
+    try:
+        import pyarrow as pa
+    except ImportError:
+        return table
+
+    fields = []
+    changed = False
+    for field in table.schema:
+        target_type = field.type
+        if pa.types.is_string_view(target_type):
+            target_type = pa.string()
+        elif pa.types.is_binary_view(target_type):
+            target_type = pa.binary()
+
+        changed = changed or target_type != field.type
+        fields.append(
+            pa.field(
+                field.name,
+                target_type,
+                nullable=field.nullable,
+                metadata=field.metadata,
+            )
+        )
+
+    if not changed:
+        return table
+
+    target_schema = pa.schema(fields, metadata=table.schema.metadata)
+    return table.cast(target_schema)
+
+
 @dataclass(slots=True)
 class ArrowTableSource:
     """Projection-aware in-memory Arrow source without eager pandas conversion."""
@@ -89,6 +122,9 @@ class ArrowTableSource:
     table: Any
     name: str = "<arrow_table>"
 
+    def __post_init__(self) -> None:
+        self.table = _normalize_arrow_table_types(self.table)
+
     def inspect(self) -> DatasetMetadata:
         return DatasetMetadata(
             name=self.name,

From cf7115f578da2187143806e0d50109b4bb32091c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:18:58 +0530
Subject: [PATCH 348/439] test: lock compute-safe Polars Arrow normalization

---
 tests/test_polars_interop.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/test_polars_interop.py b/tests/test_polars_interop.py
index 450c1da..ebbcfc8 100644
--- a/tests/test_polars_interop.py
+++ b/tests/test_polars_interop.py
@@ -1,6 +1,6 @@
 import pytest
 
-pytest.importorskip("pyarrow")
+pa = pytest.importorskip("pyarrow")
 pl = pytest.importorskip("polars")
 
 import framevitals as fv
@@ -29,6 +29,10 @@ def test_polars_dataframe_uses_generic_arrow_capsule_source():
     assert metadata.columns == frame.width
     assert metadata.supports_projection is True
     assert metadata.supports_streaming is True
+    assert not any(
+        pa.types.is_string_view(field.type) or pa.types.is_binary_view(field.type)
+        for field in source.schema()
+    )
 
     info = fv.inspect_source(frame)
     assert info == metadata.to_dict()

From 7501796d95505af569adc8d34bfa07464f81bc0a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:19:33 +0530
Subject: [PATCH 349/439] test: freeze 0.2 package-root API surface

---
 tests/test_public_surface_contract.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_public_surface_contract.py b/tests/test_public_surface_contract.py
index 699a920..e60000d 100644
--- a/tests/test_public_surface_contract.py
+++ b/tests/test_public_surface_contract.py
@@ -45,7 +45,7 @@ def test_core_public_surface_is_exported():
         "__version__",
     }
 
-    assert expected.issubset(set(fv.__all__))
+    assert set(fv.__all__) == expected
     for name in expected:
         assert hasattr(fv, name), name
 

From 98cb9ca2587f3984e7b6c49403444be7ad9698e3 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:20:01 +0530
Subject: [PATCH 350/439] fix: align compatibility facade result typing

---
 src/framevitals/api.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/framevitals/api.py b/src/framevitals/api.py
index 2b2dee4..da8bb0f 100644
--- a/src/framevitals/api.py
+++ b/src/framevitals/api.py
@@ -23,7 +23,7 @@
         GateResult,
         ValidationResult,
     )
-    from framevitals.result import AnalysisResult
+    from framevitals.result import AnalysisResult, DiagnosticResult
 
 
 DataInput = Any
@@ -36,28 +36,28 @@ def inspect_source(data: DataInput) -> dict[str, Any]:
     return _inspect_source(data)
 
 
-def profile(data: DataInput) -> dict[str, Any]:
+def profile(data: DataInput) -> DiagnosticResult:
     """Profile a dataset through the canonical focused execution path."""
     from framevitals.focused import profile as _profile
 
     return _profile(data)
 
 
-def roles(data: DataInput) -> dict[str, Any]:
+def roles(data: DataInput) -> DiagnosticResult:
     """Infer column roles through the canonical focused execution path."""
     from framevitals.focused import roles as _roles
 
     return _roles(data)
 
 
-def health(data: DataInput) -> dict[str, Any]:
+def health(data: DataInput) -> DiagnosticResult:
     """Calculate dataset health through the canonical focused execution path."""
     from framevitals.focused import health as _health
 
     return _health(data)
 
 
-def ml_readiness(data: DataInput) -> dict[str, Any]:
+def ml_readiness(data: DataInput) -> DiagnosticResult:
     """Calculate ML readiness through the canonical focused execution path."""
     from framevitals.focused import ml_readiness as _ml_readiness
 
@@ -70,7 +70,7 @@ def quality(
     max_sample_rows: int = 5_000,
     max_columns: int = 100,
     max_missingness_columns: int = 25,
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     """Run deterministic quality diagnostics through the focused engine."""
     from framevitals.focused import quality as _quality
 
@@ -87,7 +87,7 @@ def statistics(
     *,
     max_pairs: int = 20,
     mode: str = "standard",
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     """Run bounded deep statistics through the focused engine."""
     from framevitals.focused import statistics as _statistics
 
@@ -102,7 +102,7 @@ def anomalies(
     max_columns: int = 30,
     top_k: int = 25,
     mode: str = "standard",
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     """Run bounded anomaly diagnostics through the focused engine."""
     from framevitals.focused import anomalies as _anomalies
 
@@ -124,7 +124,7 @@ def relationships(
     min_abs_correlation: float = 0.80,
     max_candidate_pairs: int = 250_000,
     max_edges_returned: int = 5_000,
-) -> dict[str, Any]:
+) -> DiagnosticResult:
     """Discover strong numeric relationships through the focused engine."""
     from framevitals.focused import relationships as _relationships
 
@@ -138,7 +138,7 @@ def relationships(
     )
 
 
-def target_analysis(data: DataInput, *, target: str) -> dict[str, Any]:
+def target_analysis(data: DataInput, *, target: str) -> DiagnosticResult:
     """Run target diagnostics through the focused source-aware engine."""
     from framevitals.focused import target_analysis as _target_analysis
 

From 6e01a83b9489feef68f7b701d8d06f80e6acdfef Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:20:16 +0530
Subject: [PATCH 351/439] docs: expose complete 0.2 documentation surface

---
 mkdocs.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/mkdocs.yml b/mkdocs.yml
index d2b7709..7c247ea 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -18,7 +18,11 @@ nav:
   - Home: index.md
   - Source-aware execution: source-execution.md
   - Execution provenance: execution-provenance.md
+  - Result objects: result-objects.md
   - Quality gates and custom checks: quality-gates.md
+  - Monitoring snapshots: monitoring.md
+  - Extending FrameVitals: extending-framevitals.md
+  - Stability and compatibility: stability.md
 
 markdown_extensions:
   - admonition

From fab519cae15f2d1390b1bcf08700139d209676f9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:20:43 +0530
Subject: [PATCH 352/439] docs: align contributor setup with enforced
 guardrails

---
 CONTRIBUTING.md | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 3ec9910..4af8d37 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -14,6 +14,8 @@ python -m venv .venv
 source .venv/bin/activate
 python -m pip install --upgrade pip
 pip install -e ".[all,dev]"
+pre-commit install
+pre-commit install --hook-type pre-push
 ```
 
 On Windows PowerShell, activate the environment with:
@@ -22,7 +24,14 @@ On Windows PowerShell, activate the environment with:
 .venv\Scripts\Activate.ps1
 ```
 
-Run the test suite before making changes:
+Run the same fast local guardrails that CI enforces:
+
+```bash
+pre-commit run --all-files
+pre-commit run --all-files --hook-stage pre-push
+```
+
+Run the full test suite before opening a substantial pull request:
 
 ```bash
 pytest
@@ -78,12 +87,14 @@ The base package should not import optional stacks unless their feature is actua
 
 Current capability groups include:
 
-- `framevitals[arrow]` for Arrow-backed streaming;
+- `framevitals[arrow]` for Arrow-backed streaming and Arrow-compatible table producers;
+- `framevitals[duckdb]` for lazy DuckDB relations through Arrow transport;
 - `framevitals[excel]` for XLS/XLSX readers;
 - `framevitals[plot]` for chart/PDF plotting support;
 - `framevitals[ml]` for heavier model/explainability integrations;
 - `framevitals[ai]` for agentic AI integrations;
 - `framevitals[web]` for the Flask runtime;
+- `framevitals[docs]` for the MkDocs documentation toolchain;
 - `framevitals[all]` for all optional runtime capabilities.
 
 If a new optional dependency is introduced, prefer a lazy import plus a clear install hint. Add a dependency-boundary test when accidental eager imports would make the base install heavier or break another extra.
@@ -95,6 +106,7 @@ Useful checks include:
 ```bash
 pytest
 pytest tests/test_public_api.py
+pytest tests/test_public_surface_contract.py
 pytest tests/test_framevitals_pipeline.py
 pytest tests/test_optional_dependency_boundaries.py
 pytest tests/test_package_boundary.py

From 29eecbe12a229dbc2b973a971afdcc6486964c10 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:21:09 +0530
Subject: [PATCH 353/439] docs: harden 0.2 release and promotion checklist

---
 RELEASING.md | 68 +++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 59 insertions(+), 9 deletions(-)

diff --git a/RELEASING.md b/RELEASING.md
index 4b21ff5..31aa643 100644
--- a/RELEASING.md
+++ b/RELEASING.md
@@ -7,8 +7,36 @@ FrameVitals is configured to publish to PyPI through GitHub Actions Trusted Publ
 - `main` is the release-ready branch. Published releases and version tags come from `main`.
 - `dev` is the integration branch for ongoing development.
 - Feature and dependency-update pull requests should target `dev`.
+- Large stabilization branches should be promoted to `dev` only after their public contracts and specialized CI lanes are clean.
 - When a release is ready, promote the tested release changes from `dev` to `main`, update the version/changelog, and publish from `main`.
 
+## Stabilization branch → `dev`
+
+Promoting a development branch such as `develop/august` to `dev` is **not** a PyPI release and does not require changing the package version.
+
+Before opening or merging the promotion pull request:
+
+1. Confirm the branch is not behind `dev`, or deliberately reconcile any new `dev` commits.
+2. Confirm the package-root public surface test passes and any intended API change is documented.
+3. Confirm execution/result schema changes are additive or carry the appropriate schema-version change.
+4. Confirm `CHANGELOG.md` records user-facing changes under `Unreleased`.
+5. Confirm the core Tests and Package workflows are green.
+6. Confirm the specialized compatibility lanes relevant to the branch are green:
+   - documentation strict build;
+   - developer/pre-commit guardrails;
+   - declared minimum dependencies;
+   - Windows/macOS platform smoke;
+   - Arrow/DuckDB interoperability;
+   - Polars-through-Arrow interoperability;
+   - external check-plugin installation/discovery;
+   - native Rust bridge, when touched;
+   - reusable Gate Action smoke tests, when touched;
+   - performance guardrails for performance-sensitive changes.
+7. Confirm no known security-analysis failure is being carried into `dev`; CodeQL runs on `dev`, `main`, and their pull requests.
+8. Review the branch diff for generated files, temporary artifacts, stale compatibility shims, and accidental package-boundary changes.
+
+A large promotion pull request should summarize the public API, compatibility impact, execution-semantics changes, optional dependencies, performance implications, and remaining known limitations.
+
 ## One-time PyPI setup
 
 Before the first release:
@@ -23,16 +51,36 @@ Before the first release:
 
 A pending publisher does not reserve the package name until the first successful publication. If the repository is renamed later, update the PyPI Trusted Publisher configuration before publishing again.
 
-## Release checklist
+## `dev` → `main` release checklist
+
+Before promoting a release candidate from `dev` to `main`:
 
-1. Confirm the release commit is on `main` and the Tests and Package workflows are green.
-2. Confirm `CHANGELOG.md` contains the release notes and date.
-3. Confirm `pyproject.toml` and `src/framevitals/__init__.py` contain the same release version.
-4. Confirm the built wheel imports successfully and `framevitals --version` reports the release version.
-5. Create a GitHub release from `main` with a matching tag such as `v0.1.0`.
-6. Publishing the GitHub release triggers `.github/workflows/release.yml`.
-7. Confirm the PyPI publish workflow succeeds.
-8. Install the published package in a clean environment and run `framevitals --version`.
+1. Confirm the intended release candidate on `dev` has green core and specialized CI, including CodeQL.
+2. Convert the relevant `Unreleased` changelog entries into a dated release section and call out any breaking 0.x change explicitly.
+3. Set the release version in both `pyproject.toml` and `src/framevitals/__init__.py` and confirm they match.
+4. Build distributions from the release candidate and run `python -m twine check dist/*`.
+5. Inspect the wheel boundary: the `framevitals` package and `py.typed` must be present, while removed compatibility namespaces and repository-only files must not leak into the wheel.
+6. Install the built wheel into a clean environment and verify at minimum:
+   - `import framevitals`;
+   - installed metadata version equals `framevitals.__version__`;
+   - `framevitals --version` reports the release version.
+7. Merge the reviewed release pull request to `main` only after its required checks are green.
+
+## Publish checklist
+
+1. Confirm the release commit is on `main` and the release version/changelog are final.
+2. Create a GitHub release from that exact `main` commit with a matching tag such as `v0.2.0`.
+3. Publishing the GitHub release triggers `.github/workflows/release.yml`.
+4. The release workflow verifies that the GitHub tag, `pyproject.toml`, and `framevitals.__version__` agree before building.
+5. Confirm the Trusted Publishing job succeeds and the expected files appear on PyPI.
+6. Install the published package in a brand-new environment and run:
+
+   ```bash
+   python -c "import framevitals; print(framevitals.__version__)"
+   framevitals --version
+   ```
+
+7. Smoke-test at least one normal analysis path from the published wheel. For a minor release that changes optional capabilities, also smoke-test the affected extra in a clean environment.
 
 ## Version consistency
 
@@ -41,4 +89,6 @@ Until version metadata is centralized, keep these two values identical:
 - `pyproject.toml` → `[project].version`
 - `src/framevitals/__init__.py` → `__version__`
 
+Do not bump the version merely to merge a development/stabilization branch into `dev`. Bump it only as part of intentional release preparation.
+
 A future cleanup can derive the package version from one canonical source.

From 73aafa3b4731320aba9c2ffe306be42d2746aa5d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:24:20 +0530
Subject: [PATCH 354/439] fix: complete monitoring and system CLI surface

---
 src/framevitals/cli.py | 150 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 150 insertions(+)

diff --git a/src/framevitals/cli.py b/src/framevitals/cli.py
index 682e142..d7f3c04 100644
--- a/src/framevitals/cli.py
+++ b/src/framevitals/cli.py
@@ -134,6 +134,61 @@ def build_parser() -> argparse.ArgumentParser:
     )
     _add_output_argument(inspect_parser)
 
+    snapshot_parser = subparsers.add_parser(
+        "snapshot",
+        help="Analyze a dataset and emit compact monitoring state.",
+    )
+    snapshot_parser.add_argument("file", type=Path, help="Path to the dataset.")
+    _add_runtime_arguments(snapshot_parser)
+    snapshot_parser.add_argument(
+        "--format",
+        choices=["terminal", "json"],
+        default="terminal",
+        help="Snapshot output format.",
+    )
+    _add_output_argument(snapshot_parser)
+
+    compare_snapshots_parser = subparsers.add_parser(
+        "compare-snapshots",
+        help="Compare two compact FrameVitals monitoring snapshots.",
+    )
+    compare_snapshots_parser.add_argument(
+        "reference", type=Path, help="Reference/baseline snapshot JSON."
+    )
+    compare_snapshots_parser.add_argument(
+        "current", type=Path, help="Current snapshot JSON."
+    )
+    compare_snapshots_parser.add_argument(
+        "--format",
+        choices=["terminal", "json"],
+        default="terminal",
+        help="Snapshot-diff output format.",
+    )
+    compare_snapshots_parser.add_argument(
+        "--fail-on-change",
+        action="store_true",
+        help="Return exit code 1 when the compact snapshot state changed.",
+    )
+    _add_output_argument(compare_snapshots_parser)
+
+    system_info_parser = subparsers.add_parser(
+        "system-info",
+        help="Inspect FrameVitals CPU/native/GPU execution capabilities.",
+    )
+    system_info_parser.add_argument(
+        "--probe-gpu",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Probe GPU/CUDA availability. Use --no-probe-gpu for support-safe output.",
+    )
+    system_info_parser.add_argument(
+        "--format",
+        choices=["terminal", "json"],
+        default="terminal",
+        help="System-information output format.",
+    )
+    _add_output_argument(system_info_parser)
+
     plan_parser = subparsers.add_parser(
         "plan",
         help="Preview applicable analyses without running heavy stages.",
@@ -372,6 +427,49 @@ def _render_source(source: dict) -> str:
     return "\n".join(lines)
 
 
+def _render_snapshot(snapshot: dict) -> str:
+    state = snapshot.get("state", {})
+    source = snapshot.get("source", {})
+    dataset = state.get("dataset", {})
+    shape = dataset.get("shape", {})
+    return "\n".join([
+        "FrameVitals snapshot",
+        f"File            {source.get('filename', 'unknown')}",
+        f"Mode            {state.get('analysis_mode', 'unknown')}",
+        f"Rows            {shape.get('rows', 'unknown')}",
+        f"Columns         {shape.get('columns', 'unknown')}",
+        f"Fingerprint     {snapshot.get('fingerprint', 'unknown')}",
+    ])
+
+
+def _render_snapshot_diff(report: dict) -> str:
+    schema = report.get("schema", {})
+    findings = report.get("findings", {})
+    return "\n".join([
+        "FrameVitals snapshot diff",
+        f"Changed         {'yes' if report.get('changed') else 'no'}",
+        f"Added columns   {len(schema.get('added_columns', []))}",
+        f"Removed columns {len(schema.get('removed_columns', []))}",
+        f"Type changes    {len(schema.get('type_changes', {}))}",
+        f"New findings    {len(findings.get('new', []))}",
+        f"Resolved        {len(findings.get('resolved', []))}",
+    ])
+
+
+def _render_system_info(info: dict) -> str:
+    native = info.get("native", {})
+    gpu = info.get("gpu", {})
+    lines = [
+        "FrameVitals system info",
+        f"Python          {info.get('python', 'unknown')}",
+        f"Backend         {info.get('backend', 'unknown')}",
+        f"Native          available: {native.get('available', False)}",
+    ]
+    if gpu:
+        lines.append(f"GPU             available: {gpu.get('available', False)}")
+    return "\n".join(lines)
+
+
 def _render_validation(report: dict) -> str:
     summary = report.get("summary", {})
     lines = [
@@ -499,6 +597,58 @@ def main() -> int:
                 print(f"Source JSON    {args.output}")
         return 0
 
+    if args.command == "snapshot":
+        from framevitals.analysis_api import analyze
+
+        report = analyze(
+            args.file,
+            target=args.target,
+            mode=args.mode,
+            artifacts=False,
+            workers=args.workers,
+            preset=args.preset,
+            config=args.config,
+            disabled_modules=args.disabled_modules,
+        )
+        snapshot = report.snapshot(args.output)
+        if args.format == "json":
+            print(snapshot.to_json())
+        else:
+            print(_render_snapshot(snapshot))
+            if args.output is not None:
+                print(f"Snapshot JSON   {args.output}")
+        return 0
+
+    if args.command == "compare-snapshots":
+        from framevitals.snapshots import compare_snapshots, load_snapshot
+
+        reference = load_snapshot(args.reference)
+        current = load_snapshot(args.current)
+        result = compare_snapshots(reference, current)
+        if args.output is not None:
+            _write_json(result, args.output)
+        if args.format == "json":
+            print(json.dumps(result, indent=2, default=str))
+        else:
+            print(_render_snapshot_diff(result))
+        if args.fail_on_change and result.get("changed"):
+            return 1
+        return 0
+
+    if args.command == "system-info":
+        from framevitals.acceleration import system_info
+
+        result = system_info(probe_gpu=args.probe_gpu)
+        if args.output is not None:
+            _write_json(result, args.output)
+        if args.format == "json":
+            print(json.dumps(result, indent=2, default=str))
+        else:
+            print(_render_system_info(result))
+            if args.output is not None:
+                print(f"System JSON     {args.output}")
+        return 0
+
     if args.command == "plan":
         from framevitals.planning_api import plan
 

From 554bb22e5ac0cb17a3dce0e8dc390d2c8d2fe5a9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:27:34 +0530
Subject: [PATCH 355/439] test: respect optional AI dependency boundary

---
 tests/test_ai_agent.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/test_ai_agent.py b/tests/test_ai_agent.py
index 10f0714..b061ecb 100644
--- a/tests/test_ai_agent.py
+++ b/tests/test_ai_agent.py
@@ -1,9 +1,10 @@
 import pandas as pd
+import pytest
+
+pytest.importorskip("pydantic")
 
 import framevitals.ai_agent as ai_agent
-from framevitals.ai_agent import (
-    answer_with_agent,
-)
+from framevitals.ai_agent import answer_with_agent
 
 
 def make_result():

From eac759a01d2e9c333bd8b5e05543a0cce3d4e32a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:27:46 +0530
Subject: [PATCH 356/439] test: respect optional plotting dependency boundary

---
 tests/test_visualization_engine.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/tests/test_visualization_engine.py b/tests/test_visualization_engine.py
index af2ee4a..ff1de27 100644
--- a/tests/test_visualization_engine.py
+++ b/tests/test_visualization_engine.py
@@ -1,16 +1,13 @@
 import pandas as pd
+import pytest
 
-import framevitals.visualizer as visualizer_module
+pytest.importorskip("matplotlib")
+pytest.importorskip("seaborn")
 
-from framevitals.chart_planner import (
-    build_chart_plan,
-)
-from framevitals.column_roles import (
-    infer_column_roles,
-)
-from framevitals.profiler import (
-    build_profile,
-)
+import framevitals.visualizer as visualizer_module
+from framevitals.chart_planner import build_chart_plan
+from framevitals.column_roles import infer_column_roles
+from framevitals.profiler import build_profile
 
 
 def make_dataset():

From 8ccbd8c4dbe7e6c44fab3c2ff2af0933610d1d87 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:28:57 +0530
Subject: [PATCH 357/439] docs: record 0.2 stabilization work

---
 CHANGELOG.md | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c51612c..4bcdb3b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,17 +19,21 @@ Development continues on the development branches. Changes intended for the next
 - `framevitals plan` for previewing applicable modules, execution budgets, source capabilities, and bounded planning samples
 - `framevitals.inspect_source()` for inspecting source shape, storage kind, format, materialization state, projection support, and streaming capability without running diagnostics
 - `framevitals inspect` CLI command with terminal/JSON output and optional persisted source metadata
+- `framevitals snapshot` and `framevitals compare-snapshots` CLI commands for compact monitoring state, snapshot diffs, JSON persistence, and CI-friendly change exit codes
+- `framevitals system-info` CLI command for reproducible CPU/native/GPU capability reports in support and performance workflows
 - Compact versioned analysis snapshots with deterministic fingerprints, JSON persistence, and snapshot-to-snapshot schema/health/ML-readiness/finding diffs
 - `SnapshotHistory` for persistent compact monitoring timelines, latest/previous lookup, labels, and recent-state comparison without storing raw datasets
-- Dict-compatible result objects including `AnalysisResult`, `DriftResult`, `ValidationResult`, `CheckResult`, `GateResult`, and per-column result views
+- Dict-compatible result objects including `AnalysisResult`, `DiagnosticResult`, `DriftResult`, `ValidationResult`, `CheckResult`, `GateResult`, and per-column result views
 - `framevitals.check()` and `framevitals.run_checks()` for exact user-defined DataFrame invariants with warning/error severities and structured findings
 - Opt-in `framevitals.discover_checks()` plugin discovery through the `framevitals.checks` Python entry-point group
+- Installable third-party check-plugin example plus CI that installs, discovers, executes, and gates on external entry points
 - Focused public APIs for profiling, column roles, health, ML readiness, quality, statistics, anomalies, relationships, and target analysis without running unrelated pipeline stages
 - Source abstraction with metadata inspection, projection, and streaming capability discovery
 - Bounded Arrow-backed Parquet execution for public profiling, health, ML-readiness, roles, quality, statistics, anomaly, relationship, target-aware, drift, gate, planning, and full-analysis paths where those diagnostics support bounded execution
 - Optional Arrow-backed CSV and TSV streaming with exact row-count metadata, projected record batches, and pandas fallback for incompatible inputs
 - Native PyArrow `Table` and `RecordBatch` inputs through the same source-aware streaming engine
 - Arrow C Stream / PyCapsule-compatible table input support without requiring a library-specific adapter for every producer
+- Tested Polars DataFrame interoperability through the standard Arrow C Stream boundary without adding Polars as a FrameVitals runtime dependency
 - Optional lazy DuckDB relation source with exact count/schema metadata, projection pushdown, Arrow record batches, and full pandas materialization only for exact/full-row APIs
 - Optional `duckdb` dependency group containing DuckDB plus its Arrow transport
 - Dedicated Arrow/DuckDB interoperability CI coverage
@@ -37,11 +41,18 @@ Development continues on the development branches. Changes intended for the next
 - Optional Rust native core and Python bridge for accelerated streaming numeric state, cardinality/quantile sketches, and backend routing
 - Dedicated Arrow/NumPy-fallback and Rust/native CI coverage
 - Reproducible performance benchmark workflow with time/RSS summaries and retained JSON artifacts
+- Catastrophic performance guardrails for deterministic core and Parquet profiling scenarios with retained measurement artifacts
+- Minimum-dependency CI that installs and tests the declared core dependency floors separately from optional extras
+- Windows and macOS public-API smoke CI alongside the full Linux matrix
+- CodeQL scanning for Python and JavaScript/TypeScript on integration/release branches, pull requests, manual runs, and a scheduled cadence
 - PEP 561 `py.typed` marker for downstream type-checking support
 - Optional `excel` dependency group for XLS/XLSX readers
 - Optional `plot` dependency group for Matplotlib/Seaborn chart and report plotting support
-- Optional `docs` dependency group plus a MkDocs Material documentation site covering source-aware execution, execution provenance, quality gates, custom checks, and plugin trust boundaries
+- Optional `docs` dependency group plus a MkDocs Material documentation site covering source-aware execution, execution provenance, result objects, monitoring, quality gates, custom checks, extension authors, and stability guarantees
 - Strict path-scoped documentation CI using `mkdocs build --strict`
+- Pre-commit and pre-push contributor guardrails mirrored by dedicated CI
+- `SUPPORT.md` guidance for reproducible issue reports without requiring users to share sensitive datasets
+- Stability/compatibility documentation defining the intended `0.x` public-surface, result-schema, execution-provenance, optional-dependency, Python-version, and platform expectations
 - Contributor architecture guidance covering canonical layers, streaming provenance, optional-dependency boundaries, and performance-sensitive changes
 - Focused examples for domain-specific quality gates and persistent snapshot monitoring
 
@@ -56,10 +67,13 @@ Development continues on the development branches. Changes intended for the next
 - Top-level gate execution now reports whether any selected check family required full materialization
 - Materialization provenance is now based on actual execution semantics rather than assuming only file-backed sources can materialize
 - Focused statistics, anomaly, relationship, role, health, ML-readiness, quality, drift, validation, custom-check, and gate paths are converging on execution-provenance schema v1 without removing legacy metadata keys
-- `framevitals.api` is now a lazy compatibility facade over the canonical focused, analysis, planning, operations, check, plugin, and source-inspection engines instead of a second eager implementation
+- Focused diagnostic APIs return dict-compatible `DiagnosticResult` objects with JSON/summary/source/execution ergonomics while preserving existing mapping payloads
+- The package-root `__all__` surface is now regression-tested as an exact `0.2` stabilization contract so accidental additions and removals both fail CI
+- `framevitals.api` is now a lazy compatibility facade over the canonical focused, analysis, planning, operations, check, plugin, and source-inspection engines instead of a second eager implementation, with focused return annotations aligned to `DiagnosticResult`
 - The installed CLI now routes directly through canonical APIs and no longer relies on a runtime monkeypatch shim
 - CLI `--version` works from a no-dependencies wheel smoke install without importing the analytics stack
 - The `all` extra now includes Arrow and DuckDB interoperability capabilities
+- The `dev` extra now includes the pre-commit tool required by the documented and CI-enforced contributor workflow
 - Excel engines moved out of the default dependency set into `framevitals[excel]`; AI-only Pydantic moved into `framevitals[ai]`
 - Matplotlib and Seaborn moved out of the default dependency set into `framevitals[plot]`
 - Explainability no longer requires Matplotlib at module import time; structured importance can run without plotting and SHAP chart rendering is optional
@@ -69,6 +83,7 @@ Development continues on the development branches. Changes intended for the next
 - Project metadata now advertises the package as typed and points documentation users at the maintained `docs/` tree
 - Package CI now requires the `py.typed` marker to be present in built wheels
 - Test and package workflows now use read-only default permissions and cancel stale runs on the same branch
+- Release documentation now separates stabilization-branch promotion to `dev` from intentional `dev` → `main` versioning and PyPI publication
 - Release publishing now verifies that the GitHub release tag, `pyproject.toml` version, and `framevitals.__version__` match before building/publishing
 
 ### Fixed
@@ -79,9 +94,12 @@ Development continues on the development branches. Changes intended for the next
 - Streaming column-role cardinality metadata distinguishes bounded-sample cardinality from full-stream native estimates
 - CSV/TSV streaming sources preserve `FileSource` compatibility for existing callers and tests
 - Dedicated Arrow CI now exercises streaming statistics, drift, CSV/TSV sources, in-memory Arrow inputs, and the quality-gate CLI
+- Arrow `string_view` and `binary_view` inputs are normalized to compute-compatible Arrow types at the in-memory source boundary, fixing Polars sampling on PyArrow 25 without falling back to pandas
+- Optional AI and plotting tests now skip cleanly in core-only/minimum-dependency environments rather than requiring undeclared optional extras during test collection
 - Web runtime dependency boundaries no longer force AI/report/plot packages to import during Flask startup
 - Reusable Gate Action setup no longer relies on an invalid absolute `setup-python` cache dependency path
 - Exact validation/custom-check provenance now reports full materialization for Arrow and relation-backed inputs when they are converted to complete pandas DataFrames
+- Monitoring snapshot and system-capability CLI commands are now implemented by the installed parser/dispatcher rather than existing only as tests/documentation
 
 ## 0.1.0 - 2026-08-15
 

From 773e4175f28c1b521ff0b4237e05766d19c604ee Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:46:47 +0530
Subject: [PATCH 358/439] ci: run one-off largest scale benchmark

---
 .github/workflows/scale-benchmark-once.yml | 54 ++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 .github/workflows/scale-benchmark-once.yml

diff --git a/.github/workflows/scale-benchmark-once.yml b/.github/workflows/scale-benchmark-once.yml
new file mode 100644
index 0000000..b42f0fa
--- /dev/null
+++ b/.github/workflows/scale-benchmark-once.yml
@@ -0,0 +1,54 @@
+name: One-off Largest Scale Benchmark
+
+on:
+  push:
+    branches: [develop/august]
+    paths:
+      - .github/workflows/scale-benchmark-once.yml
+
+permissions:
+  contents: read
+
+jobs:
+  largest-profile-scale:
+    name: 100k x 105 profile stress test
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install benchmark runtime
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[arrow]"
+          python -m pip check
+
+      - name: Run largest built-in scale workload
+        run: |
+          python benchmarks/benchmark_profile_scale.py \
+            --rows 100000 \
+            --numeric-columns 100 \
+            --categorical-columns 5 \
+            --scenarios numpy auto parquet \
+            --output largest-benchmark-results.json
+
+      - name: Print benchmark findings
+        run: cat largest-benchmark-results.json
+
+      - name: Upload benchmark result
+        uses: actions/upload-artifact@v5
+        with:
+          name: framevitals-largest-benchmark
+          path: largest-benchmark-results.json
+          retention-days: 7

From 6be938b9ad61eb1913f5f3e4bee87d609c6df94e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:47:52 +0530
Subject: [PATCH 359/439] ci: remove one-off scale benchmark workflow

---
 .github/workflows/scale-benchmark-once.yml | 54 ----------------------
 1 file changed, 54 deletions(-)
 delete mode 100644 .github/workflows/scale-benchmark-once.yml

diff --git a/.github/workflows/scale-benchmark-once.yml b/.github/workflows/scale-benchmark-once.yml
deleted file mode 100644
index b42f0fa..0000000
--- a/.github/workflows/scale-benchmark-once.yml
+++ /dev/null
@@ -1,54 +0,0 @@
-name: One-off Largest Scale Benchmark
-
-on:
-  push:
-    branches: [develop/august]
-    paths:
-      - .github/workflows/scale-benchmark-once.yml
-
-permissions:
-  contents: read
-
-jobs:
-  largest-profile-scale:
-    name: 100k x 105 profile stress test
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v6
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@v6
-        with:
-          python-version: "3.12"
-          cache: pip
-          cache-dependency-path: pyproject.toml
-
-      - name: Install benchmark runtime
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[arrow]"
-          python -m pip check
-
-      - name: Run largest built-in scale workload
-        run: |
-          python benchmarks/benchmark_profile_scale.py \
-            --rows 100000 \
-            --numeric-columns 100 \
-            --categorical-columns 5 \
-            --scenarios numpy auto parquet \
-            --output largest-benchmark-results.json
-
-      - name: Print benchmark findings
-        run: cat largest-benchmark-results.json
-
-      - name: Upload benchmark result
-        uses: actions/upload-artifact@v5
-        with:
-          name: framevitals-largest-benchmark
-          path: largest-benchmark-results.json
-          retention-days: 7

From 7dd188d8bcd256aac86b9e285b455f7e813b7939 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 19:53:54 +0530
Subject: [PATCH 360/439] fix: bound streaming profile by table width

---
 src/framevitals/streaming_profile.py | 68 +++++++++++++++++++++++++---
 1 file changed, 61 insertions(+), 7 deletions(-)

diff --git a/src/framevitals/streaming_profile.py b/src/framevitals/streaming_profile.py
index 7314d88..fd6f9b4 100644
--- a/src/framevitals/streaming_profile.py
+++ b/src/framevitals/streaming_profile.py
@@ -28,6 +28,25 @@
 
 STREAM_BATCH_SIZE = 65_536
 STREAM_SAMPLE_ROWS = 50_000
+STREAM_BATCH_CELL_BUDGET = 32_000_000
+STREAM_SAMPLE_CELL_BUDGET = 6_000_000
+
+
+def _width_aware_row_limit(
+    requested_rows: int,
+    column_count: int,
+    *,
+    cell_budget: int,
+) -> int:
+    """Clamp a row budget so rows * columns stays within a bounded cell budget."""
+    if requested_rows < 1:
+        raise ValueError("requested_rows must be at least 1.")
+    if cell_budget < 1:
+        raise ValueError("cell_budget must be at least 1.")
+
+    width = max(int(column_count), 1)
+    width_limited_rows = max(int(cell_budget) // width, 1)
+    return max(1, min(int(requested_rows), width_limited_rows))
 
 
 def _require_pyarrow():
@@ -243,10 +262,22 @@ def sample_streaming_source(
     if rows < 1:
         raise ValueError(f"Dataset is empty: {metadata.name}")
 
-    positions = _sample_positions(int(rows), int(sample_rows))
+    projected_width = len(columns) if columns is not None else int(metadata.columns or 1)
+    effective_batch_size = _width_aware_row_limit(
+        int(batch_size),
+        projected_width,
+        cell_budget=STREAM_BATCH_CELL_BUDGET,
+    )
+    effective_sample_rows = _width_aware_row_limit(
+        int(sample_rows),
+        projected_width,
+        cell_budget=STREAM_SAMPLE_CELL_BUDGET,
+    )
+
+    positions = _sample_positions(int(rows), effective_sample_rows)
     frames: list[pd.DataFrame] = []
     offset = 0
-    for batch in source.iter_batches(batch_size=batch_size, columns=columns):
+    for batch in source.iter_batches(batch_size=effective_batch_size, columns=columns):
         sampled = _sample_batch(batch, positions, offset)
         if sampled is not None and not sampled.empty:
             frames.append(sampled)
@@ -330,10 +361,22 @@ def build_streaming_profile(
     schema = schema_method()
     columns = [field.name for field in schema]
     numeric_cols, categorical_cols, date_cols = _column_groups(schema)
+    numeric_col_set = set(numeric_cols)
     string_cols = _string_columns(schema)
     dtypes = {field.name: str(field.type) for field in schema}
 
-    positions = _sample_positions(int(rows), int(sample_rows))
+    width = max(len(columns), 1)
+    effective_batch_size = _width_aware_row_limit(
+        int(batch_size),
+        width,
+        cell_budget=STREAM_BATCH_CELL_BUDGET,
+    )
+    effective_sample_rows = _width_aware_row_limit(
+        int(sample_rows),
+        width,
+        cell_budget=STREAM_SAMPLE_CELL_BUDGET,
+    )
+    positions = _sample_positions(int(rows), effective_sample_rows)
     sample_frames: list[pd.DataFrame] = []
     preview_frame: pd.DataFrame | None = None
     missing_counts = {column: 0 for column in columns}
@@ -355,7 +398,7 @@ def build_streaming_profile(
 
     offset = 0
     batches_scanned = 0
-    for batch in source.iter_batches(batch_size=batch_size):
+    for batch in source.iter_batches(batch_size=effective_batch_size):
         batches_scanned += 1
         if preview_frame is None:
             preview_frame = batch.slice(0, min(15, batch.num_rows)).to_pandas()
@@ -366,7 +409,7 @@ def build_streaming_profile(
 
         for index, column in enumerate(columns):
             array = batch.column(index)
-            if column not in numeric_cols:
+            if column not in numeric_col_set:
                 missing_counts[column] += int(array.null_count)
                 string_accumulator = native_string_accumulators.get(column)
                 if string_accumulator is not None:
@@ -535,9 +578,20 @@ def build_streaming_profile(
         "streaming_metadata": {
             "enabled": True,
             "full_materialization": False,
-            "batch_size": int(batch_size),
+            "batch_size": int(effective_batch_size),
+            "requested_batch_size": int(batch_size),
+            "batch_cell_budget": int(STREAM_BATCH_CELL_BUDGET),
             "batches_scanned": int(batches_scanned),
             "sample_rows": int(len(sample)),
+            "sample_row_limit": int(effective_sample_rows),
+            "requested_sample_rows": int(sample_rows),
+            "sample_cell_budget": int(STREAM_SAMPLE_CELL_BUDGET),
+            "sample_cells_retained": int(len(sample) * len(columns)),
+            "source_columns": int(len(columns)),
+            "width_limited": bool(
+                effective_batch_size < int(batch_size)
+                or effective_sample_rows < int(sample_rows)
+            ),
             "sample_strategy": "evenly_spaced_global_rows",
             "numeric_backend": selected_backend,
             "native_string_sketch_columns": native_categorical_columns,
@@ -545,4 +599,4 @@ def build_streaming_profile(
     }
     if return_sample:
         return payload, sample
-    return payload
+    return payload
\ No newline at end of file

From b26ed9ac8261b1bc7e7271be0d6f7887be68e21b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 19:54:16 +0530
Subject: [PATCH 361/439] test: lock width-aware streaming budgets

---
 tests/test_streaming_width_budget.py | 62 ++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 tests/test_streaming_width_budget.py

diff --git a/tests/test_streaming_width_budget.py b/tests/test_streaming_width_budget.py
new file mode 100644
index 0000000..5689fd4
--- /dev/null
+++ b/tests/test_streaming_width_budget.py
@@ -0,0 +1,62 @@
+import numpy as np
+import pytest
+
+pa = pytest.importorskip("pyarrow")
+
+import framevitals
+from framevitals.streaming_profile import (
+    STREAM_BATCH_CELL_BUDGET,
+    STREAM_BATCH_SIZE,
+    STREAM_SAMPLE_CELL_BUDGET,
+    STREAM_SAMPLE_ROWS,
+    _width_aware_row_limit,
+)
+
+
+def test_width_aware_limits_preserve_normal_width_defaults():
+    columns = 105
+
+    assert _width_aware_row_limit(
+        STREAM_BATCH_SIZE,
+        columns,
+        cell_budget=STREAM_BATCH_CELL_BUDGET,
+    ) == STREAM_BATCH_SIZE
+    assert _width_aware_row_limit(
+        STREAM_SAMPLE_ROWS,
+        columns,
+        cell_budget=STREAM_SAMPLE_CELL_BUDGET,
+    ) == STREAM_SAMPLE_ROWS
+
+
+def test_width_aware_limits_clamp_ten_thousand_columns():
+    columns = 10_000
+
+    assert _width_aware_row_limit(
+        STREAM_BATCH_SIZE,
+        columns,
+        cell_budget=STREAM_BATCH_CELL_BUDGET,
+    ) == 3_200
+    assert _width_aware_row_limit(
+        STREAM_SAMPLE_ROWS,
+        columns,
+        cell_budget=STREAM_SAMPLE_CELL_BUDGET,
+    ) == 600
+
+
+def test_public_profile_discloses_width_limited_execution():
+    rows = 100
+    columns = 500
+    values = np.arange(rows, dtype=np.float64)
+    table = pa.table({f"c{index}": values + index for index in range(columns)})
+
+    result = framevitals.profile(table)
+    streaming = result["streaming_metadata"]
+
+    assert result["shape"] == {"rows": rows, "columns": columns}
+    assert streaming["full_materialization"] is False
+    assert streaming["width_limited"] is True
+    assert streaming["requested_batch_size"] == STREAM_BATCH_SIZE
+    assert streaming["batch_size"] == STREAM_BATCH_CELL_BUDGET // columns
+    assert streaming["requested_sample_rows"] == STREAM_SAMPLE_ROWS
+    assert streaming["sample_row_limit"] == STREAM_SAMPLE_CELL_BUDGET // columns
+    assert streaming["sample_cells_retained"] == rows * columns

From 1646bc57749091b5fedc467e352128d19986deb3 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 19:54:45 +0530
Subject: [PATCH 362/439] test: cover ultra-wide streaming budgets

---
 tests/test_arrow_memory.py | 56 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/tests/test_arrow_memory.py b/tests/test_arrow_memory.py
index 38dffa1..9de9bdd 100644
--- a/tests/test_arrow_memory.py
+++ b/tests/test_arrow_memory.py
@@ -5,6 +5,13 @@
 
 import framevitals
 from framevitals.sources import ArrowTableSource, resolve_source
+from framevitals.streaming_profile import (
+    STREAM_BATCH_CELL_BUDGET,
+    STREAM_BATCH_SIZE,
+    STREAM_SAMPLE_CELL_BUDGET,
+    STREAM_SAMPLE_ROWS,
+    _width_aware_row_limit,
+)
 
 
 def _table(rows: int = 6_000):
@@ -131,3 +138,52 @@ def test_arrow_record_batch_reader_is_not_silently_materialized():
 
     with pytest.raises(TypeError, match="cheap exact row count"):
         resolve_source(reader)
+
+
+def test_width_aware_limits_preserve_normal_width_defaults():
+    columns = 105
+
+    assert _width_aware_row_limit(
+        STREAM_BATCH_SIZE,
+        columns,
+        cell_budget=STREAM_BATCH_CELL_BUDGET,
+    ) == STREAM_BATCH_SIZE
+    assert _width_aware_row_limit(
+        STREAM_SAMPLE_ROWS,
+        columns,
+        cell_budget=STREAM_SAMPLE_CELL_BUDGET,
+    ) == STREAM_SAMPLE_ROWS
+
+
+def test_width_aware_limits_clamp_ten_thousand_columns():
+    columns = 10_000
+
+    assert _width_aware_row_limit(
+        STREAM_BATCH_SIZE,
+        columns,
+        cell_budget=STREAM_BATCH_CELL_BUDGET,
+    ) == 3_200
+    assert _width_aware_row_limit(
+        STREAM_SAMPLE_ROWS,
+        columns,
+        cell_budget=STREAM_SAMPLE_CELL_BUDGET,
+    ) == 600
+
+
+def test_public_profile_discloses_width_limited_execution():
+    rows = 100
+    columns = 500
+    values = np.arange(rows, dtype=np.float64)
+    table = pa.table({f"c{index}": values + index for index in range(columns)})
+
+    result = framevitals.profile(table)
+    streaming = result["streaming_metadata"]
+
+    assert result["shape"] == {"rows": rows, "columns": columns}
+    assert streaming["full_materialization"] is False
+    assert streaming["width_limited"] is True
+    assert streaming["requested_batch_size"] == STREAM_BATCH_SIZE
+    assert streaming["batch_size"] == STREAM_BATCH_CELL_BUDGET // columns
+    assert streaming["requested_sample_rows"] == STREAM_SAMPLE_ROWS
+    assert streaming["sample_row_limit"] == STREAM_SAMPLE_CELL_BUDGET // columns
+    assert streaming["sample_cells_retained"] == rows * columns

From f1ca7131daee813447d21a3d6ecae8acdb36d5fb Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 19:54:58 +0530
Subject: [PATCH 363/439] test: keep width budget coverage in Arrow suite

---
 tests/test_streaming_width_budget.py | 62 ----------------------------
 1 file changed, 62 deletions(-)
 delete mode 100644 tests/test_streaming_width_budget.py

diff --git a/tests/test_streaming_width_budget.py b/tests/test_streaming_width_budget.py
deleted file mode 100644
index 5689fd4..0000000
--- a/tests/test_streaming_width_budget.py
+++ /dev/null
@@ -1,62 +0,0 @@
-import numpy as np
-import pytest
-
-pa = pytest.importorskip("pyarrow")
-
-import framevitals
-from framevitals.streaming_profile import (
-    STREAM_BATCH_CELL_BUDGET,
-    STREAM_BATCH_SIZE,
-    STREAM_SAMPLE_CELL_BUDGET,
-    STREAM_SAMPLE_ROWS,
-    _width_aware_row_limit,
-)
-
-
-def test_width_aware_limits_preserve_normal_width_defaults():
-    columns = 105
-
-    assert _width_aware_row_limit(
-        STREAM_BATCH_SIZE,
-        columns,
-        cell_budget=STREAM_BATCH_CELL_BUDGET,
-    ) == STREAM_BATCH_SIZE
-    assert _width_aware_row_limit(
-        STREAM_SAMPLE_ROWS,
-        columns,
-        cell_budget=STREAM_SAMPLE_CELL_BUDGET,
-    ) == STREAM_SAMPLE_ROWS
-
-
-def test_width_aware_limits_clamp_ten_thousand_columns():
-    columns = 10_000
-
-    assert _width_aware_row_limit(
-        STREAM_BATCH_SIZE,
-        columns,
-        cell_budget=STREAM_BATCH_CELL_BUDGET,
-    ) == 3_200
-    assert _width_aware_row_limit(
-        STREAM_SAMPLE_ROWS,
-        columns,
-        cell_budget=STREAM_SAMPLE_CELL_BUDGET,
-    ) == 600
-
-
-def test_public_profile_discloses_width_limited_execution():
-    rows = 100
-    columns = 500
-    values = np.arange(rows, dtype=np.float64)
-    table = pa.table({f"c{index}": values + index for index in range(columns)})
-
-    result = framevitals.profile(table)
-    streaming = result["streaming_metadata"]
-
-    assert result["shape"] == {"rows": rows, "columns": columns}
-    assert streaming["full_materialization"] is False
-    assert streaming["width_limited"] is True
-    assert streaming["requested_batch_size"] == STREAM_BATCH_SIZE
-    assert streaming["batch_size"] == STREAM_BATCH_CELL_BUDGET // columns
-    assert streaming["requested_sample_rows"] == STREAM_SAMPLE_ROWS
-    assert streaming["sample_row_limit"] == STREAM_SAMPLE_CELL_BUDGET // columns
-    assert streaming["sample_cells_retained"] == rows * columns

From 680aa696fba42ea588a1442dece20027f7d3b33d Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 19:56:44 +0530
Subject: [PATCH 364/439] temp

---
 benchmarks/benchmark_wide_complex_stress.py | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 benchmarks/benchmark_wide_complex_stress.py

diff --git a/benchmarks/benchmark_wide_complex_stress.py b/benchmarks/benchmark_wide_complex_stress.py
new file mode 100644
index 0000000..c7185d1
--- /dev/null
+++ b/benchmarks/benchmark_wide_complex_stress.py
@@ -0,0 +1 @@
+# temp
\ No newline at end of file

From 75988625d40132e8bee2f85212d88be25c627bd2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 19:56:58 +0530
Subject: [PATCH 365/439] chore: remove accidental stress placeholder

---
 benchmarks/benchmark_wide_complex_stress.py | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 benchmarks/benchmark_wide_complex_stress.py

diff --git a/benchmarks/benchmark_wide_complex_stress.py b/benchmarks/benchmark_wide_complex_stress.py
deleted file mode 100644
index c7185d1..0000000
--- a/benchmarks/benchmark_wide_complex_stress.py
+++ /dev/null
@@ -1 +0,0 @@
-# temp
\ No newline at end of file

From 95d8e4f78dd70d75e3ab46cce28b09ba1f638b25 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 20:53:49 +0530
Subject: [PATCH 366/439] fix: make public analysis modes materially distinct

---
 src/framevitals/analysis_api.py | 58 ++++++++++++++++++++++++++++++---
 1 file changed, 53 insertions(+), 5 deletions(-)

diff --git a/src/framevitals/analysis_api.py b/src/framevitals/analysis_api.py
index 16f8dae..4006640 100644
--- a/src/framevitals/analysis_api.py
+++ b/src/framevitals/analysis_api.py
@@ -23,6 +23,45 @@
 DataInput = Any
 
 
+_MODE_DISABLED_MODULES: dict[str, frozenset[str]] = {
+    # Quick is intentionally an overview: profile/roles/health/readiness and
+    # lightweight quality signals only. Expensive row-dependent modules are
+    # omitted even if a target is supplied.
+    "quick": frozenset({
+        "deep_statistics",
+        "anomaly_detection",
+        "time_series",
+        "text_profile",
+        "target_intelligence",
+        "modeling",
+        "explainability",
+        "cleaning",
+    }),
+    # Standard is the operational default. It keeps practical anomaly,
+    # time-series and target diagnostics, but leaves research-grade statistics,
+    # free-text profiling and model training/explainability to deep mode.
+    "standard": frozenset({
+        "deep_statistics",
+        "text_profile",
+        "modeling",
+        "explainability",
+    }),
+    "deep": frozenset(),
+    "research": frozenset(),
+}
+
+
+def _effective_disabled_modules(
+    mode: str,
+    user_disabled: tuple[str, ...],
+) -> tuple[str, ...]:
+    """Merge explicit disables with the stable module policy for a mode."""
+    implicit = _MODE_DISABLED_MODULES.get(mode)
+    if implicit is None:
+        raise ValueError(f"Unknown analysis mode: {mode}")
+    return tuple(sorted(set(user_disabled) | set(implicit)))
+
+
 def analyze(
     data: DataInput,
     *,
@@ -44,6 +83,11 @@ def analyze(
         workers=workers,
         disabled_modules=disabled_modules,
     )
+    mode_disabled = tuple(sorted(_MODE_DISABLED_MODULES[resolved.mode]))
+    effective_disabled = _effective_disabled_modules(
+        resolved.mode,
+        resolved.disabled_modules,
+    )
     dataset_id = f"fv_{uuid4().hex[:12]}"
 
     # Preserve the direct DataFrame path so callers do not pay for a defensive
@@ -60,7 +104,7 @@ def analyze(
             skip_ai=True,
             dataframe=data,
             write_artifacts=resolved.artifacts,
-            disabled_modules=resolved.disabled_modules,
+            disabled_modules=effective_disabled,
         )
     else:
         source = resolve_source(data)
@@ -83,7 +127,7 @@ def analyze(
                 target_column=resolved.target,
                 parallel_workers=resolved.workers,
                 skip_ai=True,
-                disabled_modules=resolved.disabled_modules,
+                disabled_modules=effective_disabled,
             )
         elif isinstance(data, (str, Path)):
             path = Path(data)
@@ -100,7 +144,7 @@ def analyze(
                 parallel_workers=resolved.workers,
                 skip_ai=True,
                 write_artifacts=resolved.artifacts,
-                disabled_modules=resolved.disabled_modules,
+                disabled_modules=effective_disabled,
             )
         else:
             dataframe = source.load()
@@ -113,8 +157,12 @@ def analyze(
                 skip_ai=True,
                 dataframe=dataframe,
                 write_artifacts=resolved.artifacts,
-                disabled_modules=resolved.disabled_modules,
+                disabled_modules=effective_disabled,
             )
 
-    payload["config"] = resolved.to_dict()
+    payload["config"] = {
+        **resolved.to_dict(),
+        "mode_disabled_modules": list(mode_disabled),
+        "effective_disabled_modules": list(effective_disabled),
+    }
     return AnalysisResult(payload)

From 1cc3e3f76b63e43ec2bd9117c3553a4f346623b5 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 20:54:01 +0530
Subject: [PATCH 367/439] test: lock public analysis mode policy

---
 tests/test_analysis_modes.py | 103 +++++++++++++++++++++++++++++++++++
 1 file changed, 103 insertions(+)
 create mode 100644 tests/test_analysis_modes.py

diff --git a/tests/test_analysis_modes.py b/tests/test_analysis_modes.py
new file mode 100644
index 0000000..12144c2
--- /dev/null
+++ b/tests/test_analysis_modes.py
@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import pandas as pd
+import pytest
+
+import framevitals.analysis_api as analysis_api
+
+
+@pytest.mark.parametrize(
+    ("mode", "expected_mode_disabled"),
+    [
+        (
+            "quick",
+            {
+                "deep_statistics",
+                "anomaly_detection",
+                "time_series",
+                "text_profile",
+                "target_intelligence",
+                "modeling",
+                "explainability",
+                "cleaning",
+            },
+        ),
+        (
+            "standard",
+            {"deep_statistics", "text_profile", "modeling", "explainability"},
+        ),
+        ("deep", set()),
+        ("research", set()),
+    ],
+)
+def test_mode_policy_is_explicit_and_stable(mode, expected_mode_disabled):
+    assert set(analysis_api._MODE_DISABLED_MODULES[mode]) == expected_mode_disabled
+
+
+def test_effective_mode_policy_preserves_explicit_user_disables():
+    disabled = analysis_api._effective_disabled_modules(
+        "standard",
+        ("time_series", "charts"),
+    )
+    assert set(disabled) == {
+        "deep_statistics",
+        "text_profile",
+        "modeling",
+        "explainability",
+        "time_series",
+        "charts",
+    }
+
+
+def test_standard_public_analysis_does_not_schedule_deep_only_modules(monkeypatch):
+    captured: dict[str, object] = {}
+
+    def fake_run_full_analysis(**kwargs):
+        captured.update(kwargs)
+        return {
+            "filename": "<dataframe>",
+            "profile": {"shape": {"rows": 3, "columns": 2}},
+            "execution": {},
+        }
+
+    monkeypatch.setattr(analysis_api, "run_full_analysis", fake_run_full_analysis)
+    result = analysis_api.analyze(
+        pd.DataFrame({"x": [1, 2, 3], "target": [0, 1, 0]}),
+        mode="standard",
+        target="target",
+        artifacts=False,
+    )
+
+    disabled = set(captured["disabled_modules"])
+    assert {"deep_statistics", "text_profile", "modeling", "explainability"} <= disabled
+    assert "anomaly_detection" not in disabled
+    assert "time_series" not in disabled
+    assert "target_intelligence" not in disabled
+    assert set(result["config"]["mode_disabled_modules"]) == {
+        "deep_statistics",
+        "text_profile",
+        "modeling",
+        "explainability",
+    }
+
+
+def test_deep_public_analysis_keeps_deep_modules_enabled(monkeypatch):
+    captured: dict[str, object] = {}
+
+    def fake_run_full_analysis(**kwargs):
+        captured.update(kwargs)
+        return {
+            "filename": "<dataframe>",
+            "profile": {"shape": {"rows": 3, "columns": 2}},
+            "execution": {},
+        }
+
+    monkeypatch.setattr(analysis_api, "run_full_analysis", fake_run_full_analysis)
+    analysis_api.analyze(
+        pd.DataFrame({"x": [1, 2, 3], "target": [0, 1, 0]}),
+        mode="deep",
+        target="target",
+        artifacts=False,
+    )
+
+    assert tuple(captured["disabled_modules"]) == ()

From 89a027484b0b35cb076cd5d9d683c4477339748a Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 20:54:29 +0530
Subject: [PATCH 368/439] fix: align selector inventory with standard mode
 execution

---
 src/framevitals/analysis_inventory.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/analysis_inventory.py b/src/framevitals/analysis_inventory.py
index 3a8a250..769106a 100644
--- a/src/framevitals/analysis_inventory.py
+++ b/src/framevitals/analysis_inventory.py
@@ -88,7 +88,7 @@
      "requires": {}, "outputs": ["warnings", "severity"]},
 
     {"id": "target_analysis", "name": "Target Column Analysis", "category": "Machine Learning",
-     "modes": ["deep", "research"], "priority": "high",
+     "modes": ["standard", "deep", "research"], "priority": "high",
      "requires": {}, "requires_user_target": True, "outputs": ["target_candidates", "task_type"]},
 
     {"id": "feature_importance", "name": "Feature Importance Analysis", "category": "Machine Learning",
@@ -113,7 +113,7 @@
      "outputs": ["privacy_score", "pii_warning"]},
 
     {"id": "time_series_signal", "name": "Time Series Structure Signal", "category": "Temporal",
-     "modes": ["deep", "research"], "priority": "medium",
+     "modes": ["standard", "deep", "research"], "priority": "medium",
      "requires": {"has_time_series_structure": True}, "outputs": ["warning", "recommendation"]},
 
     {"id": "ai_summary", "name": "AI Insight Generation", "category": "AI",

From 1a436f07845313832ee5365658bad9383bf84882 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 20:55:43 +0530
Subject: [PATCH 369/439] test: align selector contract with mode execution

---
 tests/test_analysis_modes.py | 47 ++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/tests/test_analysis_modes.py b/tests/test_analysis_modes.py
index 12144c2..e8a88ba 100644
--- a/tests/test_analysis_modes.py
+++ b/tests/test_analysis_modes.py
@@ -4,6 +4,7 @@
 import pytest
 
 import framevitals.analysis_api as analysis_api
+from framevitals.analysis_selector import select_analyses
 
 
 @pytest.mark.parametrize(
@@ -101,3 +102,49 @@ def fake_run_full_analysis(**kwargs):
     )
 
     assert tuple(captured["disabled_modules"]) == ()
+
+
+def test_selector_matches_standard_and_deep_mode_contracts():
+    signals = {
+        "row_count": 8_000,
+        "has_numeric_columns": True,
+        "has_multiple_numeric_columns": True,
+        "has_categorical_columns": True,
+        "has_long_text_columns": True,
+        "has_datetime_columns": True,
+        "has_time_series_structure": True,
+        "has_id_like_columns": True,
+        "has_high_missingness": True,
+        "has_sensitive_column_candidates": True,
+        "has_email_like_columns": True,
+    }
+
+    standard = select_analyses(
+        signals,
+        analysis_mode="standard",
+        target_column="target",
+    )
+    deep = select_analyses(
+        signals,
+        analysis_mode="deep",
+        target_column="target",
+    )
+
+    standard_ids = {item["id"] for item in standard["selected_analyses"]}
+    deep_ids = {item["id"] for item in deep["selected_analyses"]}
+
+    assert {"target_analysis", "time_series_signal"} <= standard_ids
+    assert {
+        "normality_tests",
+        "chi_square_analysis",
+        "text_analysis",
+        "feature_importance",
+        "baseline_model",
+    } <= deep_ids
+    assert {
+        "normality_tests",
+        "chi_square_analysis",
+        "text_analysis",
+        "feature_importance",
+        "baseline_model",
+    }.isdisjoint(standard_ids)

From 4cfa0ea3fa0bc86f73fcd9787a6153c32f0051e7 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 20:58:18 +0530
Subject: [PATCH 370/439] test: align quick streaming expectations with mode
 policy

---
 tests/test_parquet_streaming.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
index 609f669..89d8b97 100644
--- a/tests/test_parquet_streaming.py
+++ b/tests/test_parquet_streaming.py
@@ -87,7 +87,8 @@ def fail_load(self):
     assert result["execution"]["streaming"]["working_sample_rows"] == 5_000
     assert result["execution"]["streaming"]["single_full_source_profile_scan"] is True
     assert result["execution"]["module_scope"]["profile"] == "full_stream"
-    assert result["cleaning"]["streaming_status"] == "deferred_streaming"
+    assert result["cleaning"]["streaming_status"] == "disabled"
+    assert result["execution"]["module_status"]["cleaning"] == "disabled"
     assert result["charts"] == []
     assert result["config"]["artifacts"] is False
 

From ad5d6939f4f8c110cd43c5beb9ec519c8078cdf2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 21:00:42 +0530
Subject: [PATCH 371/439] fix: preserve quick target and artifact compatibility

---
 src/framevitals/analysis_api.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/src/framevitals/analysis_api.py b/src/framevitals/analysis_api.py
index 4006640..93fabbb 100644
--- a/src/framevitals/analysis_api.py
+++ b/src/framevitals/analysis_api.py
@@ -24,18 +24,16 @@
 
 
 _MODE_DISABLED_MODULES: dict[str, frozenset[str]] = {
-    # Quick is intentionally an overview: profile/roles/health/readiness and
-    # lightweight quality signals only. Expensive row-dependent modules are
-    # omitted even if a target is supplied.
+    # Quick is intentionally an overview. Keep explicit target intelligence and
+    # artifact cleaning available for backwards compatibility, while omitting
+    # the heavier anomaly/time-series/research/modeling layers.
     "quick": frozenset({
         "deep_statistics",
         "anomaly_detection",
         "time_series",
         "text_profile",
-        "target_intelligence",
         "modeling",
         "explainability",
-        "cleaning",
     }),
     # Standard is the operational default. It keeps practical anomaly,
     # time-series and target diagnostics, but leaves research-grade statistics,
@@ -83,7 +81,6 @@ def analyze(
         workers=workers,
         disabled_modules=disabled_modules,
     )
-    mode_disabled = tuple(sorted(_MODE_DISABLED_MODULES[resolved.mode]))
     effective_disabled = _effective_disabled_modules(
         resolved.mode,
         resolved.disabled_modules,
@@ -160,9 +157,8 @@ def analyze(
                 disabled_modules=effective_disabled,
             )
 
-    payload["config"] = {
-        **resolved.to_dict(),
-        "mode_disabled_modules": list(mode_disabled),
-        "effective_disabled_modules": list(effective_disabled),
-    }
+    # Keep the established public config schema stable. Effective mode policy is
+    # already visible through execution.module_status and does not need to mutate
+    # the versioned config payload.
+    payload["config"] = resolved.to_dict()
     return AnalysisResult(payload)

From 40171b55d2568874407251b37b3b9e6178622238 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 21:01:09 +0530
Subject: [PATCH 372/439] test: preserve quick compatibility in mode contract

---
 tests/test_analysis_modes.py | 42 +++++++++++++++++++++++++++++-------
 1 file changed, 34 insertions(+), 8 deletions(-)

diff --git a/tests/test_analysis_modes.py b/tests/test_analysis_modes.py
index e8a88ba..2a906a5 100644
--- a/tests/test_analysis_modes.py
+++ b/tests/test_analysis_modes.py
@@ -17,10 +17,8 @@
                 "anomaly_detection",
                 "time_series",
                 "text_profile",
-                "target_intelligence",
                 "modeling",
                 "explainability",
-                "cleaning",
             },
         ),
         (
@@ -50,6 +48,38 @@ def test_effective_mode_policy_preserves_explicit_user_disables():
     }
 
 
+def test_quick_keeps_target_intelligence_and_cleaning_available(monkeypatch):
+    captured: dict[str, object] = {}
+
+    def fake_run_full_analysis(**kwargs):
+        captured.update(kwargs)
+        return {
+            "filename": "<dataframe>",
+            "profile": {"shape": {"rows": 3, "columns": 2}},
+            "execution": {},
+        }
+
+    monkeypatch.setattr(analysis_api, "run_full_analysis", fake_run_full_analysis)
+    result = analysis_api.analyze(
+        pd.DataFrame({"x": [1, 2, 3], "target": [0, 1, 0]}),
+        mode="quick",
+        target="target",
+        artifacts=True,
+    )
+
+    disabled = set(captured["disabled_modules"])
+    assert "target_intelligence" not in disabled
+    assert "cleaning" not in disabled
+    assert {"deep_statistics", "anomaly_detection", "time_series", "text_profile", "modeling", "explainability"} <= disabled
+    assert result["config"] == {
+        "mode": "quick",
+        "target": "target",
+        "artifacts": True,
+        "workers": result["config"]["workers"],
+        "disabled_modules": (),
+    }
+
+
 def test_standard_public_analysis_does_not_schedule_deep_only_modules(monkeypatch):
     captured: dict[str, object] = {}
 
@@ -74,12 +104,8 @@ def fake_run_full_analysis(**kwargs):
     assert "anomaly_detection" not in disabled
     assert "time_series" not in disabled
     assert "target_intelligence" not in disabled
-    assert set(result["config"]["mode_disabled_modules"]) == {
-        "deep_statistics",
-        "text_profile",
-        "modeling",
-        "explainability",
-    }
+    assert result["config"]["mode"] == "standard"
+    assert result["config"]["disabled_modules"] == ()
 
 
 def test_deep_public_analysis_keeps_deep_modules_enabled(monkeypatch):

From 12c5056f2e2667cc743e6353b81cc11c0f948ace Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 21:02:07 +0530
Subject: [PATCH 373/439] test: restore deferred quick streaming cleaning
 contract

---
 tests/test_parquet_streaming.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
index 89d8b97..609f669 100644
--- a/tests/test_parquet_streaming.py
+++ b/tests/test_parquet_streaming.py
@@ -87,8 +87,7 @@ def fail_load(self):
     assert result["execution"]["streaming"]["working_sample_rows"] == 5_000
     assert result["execution"]["streaming"]["single_full_source_profile_scan"] is True
     assert result["execution"]["module_scope"]["profile"] == "full_stream"
-    assert result["cleaning"]["streaming_status"] == "disabled"
-    assert result["execution"]["module_status"]["cleaning"] == "disabled"
+    assert result["cleaning"]["streaming_status"] == "deferred_streaming"
     assert result["charts"] == []
     assert result["config"]["artifacts"] is False
 

From 1be238e06da353c033d16ed4b59dda7b3b9d296e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 21:04:47 +0530
Subject: [PATCH 374/439] fix: preserve explicit disabled-module provenance

---
 src/framevitals/analysis_api.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/framevitals/analysis_api.py b/src/framevitals/analysis_api.py
index 93fabbb..f4c9511 100644
--- a/src/framevitals/analysis_api.py
+++ b/src/framevitals/analysis_api.py
@@ -157,8 +157,12 @@ def analyze(
                 disabled_modules=effective_disabled,
             )
 
-    # Keep the established public config schema stable. Effective mode policy is
-    # already visible through execution.module_status and does not need to mutate
-    # the versioned config payload.
+    # Keep public configuration/provenance compatible: ``disabled_modules``
+    # describes only explicit caller configuration. Mode policy is observable
+    # through ``execution.module_status`` and analysis selection instead of
+    # masquerading as a user-supplied disable list.
     payload["config"] = resolved.to_dict()
+    execution = payload.get("execution")
+    if isinstance(execution, dict):
+        execution["disabled_modules"] = sorted(resolved.disabled_modules)
     return AnalysisResult(payload)

From f01b3c6ff90f98df61c66f53b57126f0b67e0968 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 21:19:29 +0530
Subject: [PATCH 375/439] perf: reserve model training for research mode

---
 src/framevitals/analysis_api.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/analysis_api.py b/src/framevitals/analysis_api.py
index f4c9511..90e0521 100644
--- a/src/framevitals/analysis_api.py
+++ b/src/framevitals/analysis_api.py
@@ -37,14 +37,17 @@
     }),
     # Standard is the operational default. It keeps practical anomaly,
     # time-series and target diagnostics, but leaves research-grade statistics,
-    # free-text profiling and model training/explainability to deep mode.
+    # free-text profiling and model training/explainability to deeper tiers.
     "standard": frozenset({
         "deep_statistics",
         "text_profile",
         "modeling",
         "explainability",
     }),
-    "deep": frozenset(),
+    # Deep is the advanced diagnostic tier: research-grade statistics and text
+    # profiling are enabled, while repeated model CV/refitting is reserved for
+    # research mode where the extra runtime is an explicit user choice.
+    "deep": frozenset({"modeling", "explainability"}),
     "research": frozenset(),
 }
 

From 62472662d2562e582f68d5d37b1167c6ac21f03f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 21:19:55 +0530
Subject: [PATCH 376/439] fix: align research-only modeling inventory

---
 src/framevitals/analysis_inventory.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/framevitals/analysis_inventory.py b/src/framevitals/analysis_inventory.py
index 769106a..aee09c6 100644
--- a/src/framevitals/analysis_inventory.py
+++ b/src/framevitals/analysis_inventory.py
@@ -92,11 +92,11 @@
      "requires": {}, "requires_user_target": True, "outputs": ["target_candidates", "task_type"]},
 
     {"id": "feature_importance", "name": "Feature Importance Analysis", "category": "Machine Learning",
-     "modes": ["deep", "research"], "priority": "high",
+     "modes": ["research"], "priority": "high",
      "requires": {"has_numeric_columns": True}, "requires_user_target": True, "outputs": ["ranking", "chart"]},
 
     {"id": "baseline_model", "name": "Baseline Model Analysis", "category": "Machine Learning",
-     "modes": ["deep", "research"], "priority": "high",
+     "modes": ["research"], "priority": "high",
      "requires": {"has_numeric_columns": True}, "requires_user_target": True, "outputs": ["metrics"]},
 
     {"id": "anomaly_detection", "name": "Anomaly Detection", "category": "Anomaly",

From fa40064d8745a75c8eeccc44e8e7bb8322902963 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 21:20:18 +0530
Subject: [PATCH 377/439] test: lock four-tier analysis mode contract

---
 tests/test_analysis_modes.py | 73 ++++++++++++++++++++++--------------
 1 file changed, 45 insertions(+), 28 deletions(-)

diff --git a/tests/test_analysis_modes.py b/tests/test_analysis_modes.py
index 2a906a5..10cf218 100644
--- a/tests/test_analysis_modes.py
+++ b/tests/test_analysis_modes.py
@@ -25,7 +25,7 @@
             "standard",
             {"deep_statistics", "text_profile", "modeling", "explainability"},
         ),
-        ("deep", set()),
+        ("deep", {"modeling", "explainability"}),
         ("research", set()),
     ],
 )
@@ -70,7 +70,14 @@ def fake_run_full_analysis(**kwargs):
     disabled = set(captured["disabled_modules"])
     assert "target_intelligence" not in disabled
     assert "cleaning" not in disabled
-    assert {"deep_statistics", "anomaly_detection", "time_series", "text_profile", "modeling", "explainability"} <= disabled
+    assert {
+        "deep_statistics",
+        "anomaly_detection",
+        "time_series",
+        "text_profile",
+        "modeling",
+        "explainability",
+    } <= disabled
     assert result["config"] == {
         "mode": "quick",
         "target": "target",
@@ -108,7 +115,7 @@ def fake_run_full_analysis(**kwargs):
     assert result["config"]["disabled_modules"] == ()
 
 
-def test_deep_public_analysis_keeps_deep_modules_enabled(monkeypatch):
+def test_deep_keeps_advanced_diagnostics_but_reserves_modeling_for_research(monkeypatch):
     captured: dict[str, object] = {}
 
     def fake_run_full_analysis(**kwargs):
@@ -127,10 +134,35 @@ def fake_run_full_analysis(**kwargs):
         artifacts=False,
     )
 
+    disabled = set(captured["disabled_modules"])
+    assert disabled == {"modeling", "explainability"}
+    assert "deep_statistics" not in disabled
+    assert "text_profile" not in disabled
+
+
+def test_research_keeps_modeling_and_explainability_enabled(monkeypatch):
+    captured: dict[str, object] = {}
+
+    def fake_run_full_analysis(**kwargs):
+        captured.update(kwargs)
+        return {
+            "filename": "<dataframe>",
+            "profile": {"shape": {"rows": 3, "columns": 2}},
+            "execution": {},
+        }
+
+    monkeypatch.setattr(analysis_api, "run_full_analysis", fake_run_full_analysis)
+    analysis_api.analyze(
+        pd.DataFrame({"x": [1, 2, 3], "target": [0, 1, 0]}),
+        mode="research",
+        target="target",
+        artifacts=False,
+    )
+
     assert tuple(captured["disabled_modules"]) == ()
 
 
-def test_selector_matches_standard_and_deep_mode_contracts():
+def test_selector_matches_standard_deep_and_research_contracts():
     signals = {
         "row_count": 8_000,
         "has_numeric_columns": True,
@@ -145,32 +177,17 @@ def test_selector_matches_standard_and_deep_mode_contracts():
         "has_email_like_columns": True,
     }
 
-    standard = select_analyses(
-        signals,
-        analysis_mode="standard",
-        target_column="target",
-    )
-    deep = select_analyses(
-        signals,
-        analysis_mode="deep",
-        target_column="target",
-    )
+    standard = select_analyses(signals, analysis_mode="standard", target_column="target")
+    deep = select_analyses(signals, analysis_mode="deep", target_column="target")
+    research = select_analyses(signals, analysis_mode="research", target_column="target")
 
     standard_ids = {item["id"] for item in standard["selected_analyses"]}
     deep_ids = {item["id"] for item in deep["selected_analyses"]}
+    research_ids = {item["id"] for item in research["selected_analyses"]}
 
     assert {"target_analysis", "time_series_signal"} <= standard_ids
-    assert {
-        "normality_tests",
-        "chi_square_analysis",
-        "text_analysis",
-        "feature_importance",
-        "baseline_model",
-    } <= deep_ids
-    assert {
-        "normality_tests",
-        "chi_square_analysis",
-        "text_analysis",
-        "feature_importance",
-        "baseline_model",
-    }.isdisjoint(standard_ids)
+    assert {"normality_tests", "chi_square_analysis", "text_analysis"} <= deep_ids
+    assert {"feature_importance", "baseline_model"} <= research_ids
+
+    assert {"normality_tests", "chi_square_analysis", "text_analysis"}.isdisjoint(standard_ids)
+    assert {"feature_importance", "baseline_model"}.isdisjoint(deep_ids)

From f020f3fe3cdd81b591649ed81c6de6f8a469d5dd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Sun, 16 Aug 2026 21:23:14 +0530
Subject: [PATCH 378/439] test: align planning expectations with analysis tiers

---
 tests/test_planning.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/tests/test_planning.py b/tests/test_planning.py
index e9bfa59..a8b2022 100644
--- a/tests/test_planning.py
+++ b/tests/test_planning.py
@@ -27,14 +27,20 @@ def test_public_plan_returns_explainable_dict_compatible_plan():
     assert "preview only" in result.explain_text().lower()
 
 
-def test_deep_plan_with_target_selects_target_aware_work():
-    result = framevitals.plan(_dataset(), mode="deep", target="churn")
-    selected_ids = {item["id"] for item in result.selected}
-
-    assert "target_analysis" in selected_ids
-    assert "baseline_model" in selected_ids
-    assert "feature_importance" in selected_ids
-    assert result["config"]["target"] == "churn"
+def test_deep_plan_keeps_target_analysis_and_reserves_modeling_for_research():
+    deep = framevitals.plan(_dataset(), mode="deep", target="churn")
+    deep_ids = {item["id"] for item in deep.selected}
+
+    assert "target_analysis" in deep_ids
+    assert "baseline_model" not in deep_ids
+    assert "feature_importance" not in deep_ids
+    assert deep["config"]["target"] == "churn"
+
+    research = framevitals.plan(_dataset(), mode="research", target="churn")
+    research_ids = {item["id"] for item in research.selected}
+
+    assert {"target_analysis", "baseline_model", "feature_importance"} <= research_ids
+    assert research["config"]["target"] == "churn"
 
 
 def test_plan_uses_config_without_writing_artifacts(tmp_path, monkeypatch):

From 59b9aeec7b72a1a26e74fe283cd18f2fcc784b3e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 00:36:51 +0530
Subject: [PATCH 379/439] feat: add adaptive streaming intelligence

Add adaptive deep-analysis column triage, fast robust/random-projection anomaly screening for Standard/Deep, Research-only bounded neural reconstruction anomalies, adaptive NumPy full-stream log-quantile sketches, and bounded Page-Hinkley mean-shift detection.

Preserve public execution scope compatibility and ultra-wide safety fallbacks.

Validated with full core and Arrow suites, platform/interoperability/lower-bound/performance CI, and the same 8k x 180 Deep benchmark improving from ~76.17s to 12.779s.
---
 src/framevitals/budgeted_analysis.py  | 141 ++++++++++---
 src/framevitals/deep_triage.py        | 166 +++++++++++++++
 src/framevitals/fast_anomaly.py       | 277 ++++++++++++++++++++++++++
 src/framevitals/neural_anomaly.py     | 138 +++++++++++++
 src/framevitals/stream_change.py      | 174 ++++++++++++++++
 src/framevitals/streaming_profile.py  |  84 ++++++--
 src/framevitals/streaming_sketches.py | 155 ++++++++++++++
 tests/test_adaptive_execution.py      |  32 ++-
 tests/test_arrow_memory.py            |  31 +++
 tests/test_deep_triage.py             |  70 +++++++
 tests/test_fast_anomaly.py            |  37 ++++
 tests/test_neural_anomaly.py          |  66 ++++++
 tests/test_stream_change.py           |  96 +++++++++
 tests/test_streaming_sketches.py      |  44 ++++
 14 files changed, 1462 insertions(+), 49 deletions(-)
 create mode 100644 src/framevitals/deep_triage.py
 create mode 100644 src/framevitals/fast_anomaly.py
 create mode 100644 src/framevitals/neural_anomaly.py
 create mode 100644 src/framevitals/stream_change.py
 create mode 100644 src/framevitals/streaming_sketches.py
 create mode 100644 tests/test_deep_triage.py
 create mode 100644 tests/test_fast_anomaly.py
 create mode 100644 tests/test_neural_anomaly.py
 create mode 100644 tests/test_stream_change.py
 create mode 100644 tests/test_streaming_sketches.py

diff --git a/src/framevitals/budgeted_analysis.py b/src/framevitals/budgeted_analysis.py
index a5b1638..b8f50c9 100644
--- a/src/framevitals/budgeted_analysis.py
+++ b/src/framevitals/budgeted_analysis.py
@@ -1,9 +1,8 @@
-"""Resource-bounded adapters around legacy analysis modules.
+"""Resource-bounded adapters around expensive analysis modules.
 
-These adapters are a migration bridge. They make existing analyses safe on
-large inputs today while FrameVitals evolves toward streaming Rust/Arrow kernels
-and per-operation exact/approximate execution. Sampling is deterministic and
-always included in returned metadata.
+These adapters make legacy analyses safe on large inputs while FrameVitals moves
+more work into streaming/native kernels. Sampling and adaptive column selection
+are deterministic and are always disclosed in returned execution metadata.
 """
 
 from __future__ import annotations
@@ -14,8 +13,12 @@
 
 from framevitals.anomaly_ensemble import detect_anomalies_ensemble
 from framevitals.deep_statistics_v2 import run_deep_statistics_v2
+from framevitals.deep_triage import triage_deep_columns
 from framevitals.execution import ExecutionBudget, deterministic_sample_frame
+from framevitals.fast_anomaly import fast_anomaly_scan
+from framevitals.neural_anomaly import neural_reconstruction_anomalies
 from framevitals.provenance import normalize_execution
+from framevitals.stream_change import scan_ordered_mean_shift
 from framevitals.time_series import detect_and_analyze_time_series
 
 
@@ -45,12 +48,7 @@ def run_budgeted_deep_statistics(
     budget: ExecutionBudget,
     max_pairs: int | None = None,
 ) -> dict[str, Any]:
-    """Run legacy deep statistics without exposing it to unbounded row counts.
-
-    The current v2 implementation performs several operations over one shared
-    frame, including SciPy BCa bootstrap. Until those operations are separated,
-    the strictest relevant row budget is used for the shared frame.
-    """
+    """Run deep statistics on a bounded, adaptively selected diagnostic view."""
     sample_limit = max(
         20,
         min(
@@ -61,6 +59,14 @@ def run_budgeted_deep_statistics(
     sample_limit = min(sample_limit, max(len(dataframe), 1))
     work, sampling = deterministic_sample_frame(dataframe, sample_limit)
 
+    triage = triage_deep_columns(work, mode=budget.mode)
+    selected_columns = list(triage.selected_columns)
+    diagnostic_view = (
+        work.loc[:, selected_columns]
+        if selected_columns
+        else pd.DataFrame(index=work.index)
+    )
+
     pair_budget = (
         budget.relationship_pair_budget
         if max_pairs is None
@@ -69,16 +75,21 @@ def run_budgeted_deep_statistics(
     if pair_budget < 1:
         raise ValueError("max_pairs must be at least 1.")
 
-    payload = run_deep_statistics_v2(work, max_pairs=pair_budget)
+    payload = run_deep_statistics_v2(diagnostic_view, max_pairs=pair_budget)
+    triage_payload = triage.to_dict()
+    payload["column_triage"] = triage_payload
+
     sampling = {
         **sampling,
         "reason": (
-            "Legacy deep-statistics operations share one bounded frame; "
-            "the strict bootstrap budget is applied to prevent quadratic BCa memory growth."
-            if sampling["sampled"]
-            else "Full input fits within the deep-statistics execution budget."
+            "Deep statistics use a bounded row view plus adaptive column triage; "
+            "bootstrap/distribution work is reserved for the highest-interest columns."
         ),
         "pair_budget": int(pair_budget),
+        "column_triage": triage_payload,
+        "source_columns": int(dataframe.shape[1]),
+        "diagnostic_columns": int(len(selected_columns)),
+        "adaptive_strategy": "column_interest_triage",
     }
     return _attach_execution(
         payload,
@@ -97,25 +108,55 @@ def run_budgeted_anomalies(
     max_columns: int = 30,
     top_k: int = 25,
 ) -> dict[str, Any]:
-    """Run the anomaly ensemble on a bounded representative frame when needed."""
+    """Run fast screening in standard/deep and full confirmation in research."""
     sample_limit = max(20, min(budget.anomaly_sample_rows, max(len(dataframe), 1)))
     work, sampling = deterministic_sample_frame(dataframe, sample_limit)
-    payload = detect_anomalies_ensemble(
-        work,
-        contamination=contamination,
-        threshold=threshold,
-        max_columns=max_columns,
-        top_k=top_k,
-    )
+
+    if budget.mode == "research":
+        payload = detect_anomalies_ensemble(
+            work,
+            contamination=contamination,
+            threshold=threshold,
+            max_columns=max_columns,
+            top_k=top_k,
+        )
+        anomaly_strategy = "classical_ensemble_plus_neural_reconstruction"
+        try:
+            payload["neural_reconstruction"] = neural_reconstruction_anomalies(
+                work,
+                max_rows=min(3_000, max(len(work), 20)),
+                max_columns=min(max_columns, 24),
+                max_iter=35,
+                top_k=top_k,
+            )
+        except Exception as exc:  # neural diagnostics must fail soft
+            payload["neural_reconstruction"] = {
+                "available": False,
+                "reason": f"{type(exc).__name__}: {exc}",
+            }
+    else:
+        payload = fast_anomaly_scan(
+            work,
+            contamination=contamination,
+            threshold=threshold,
+            max_columns=min(max_columns, 24),
+            projections=12,
+            top_k=top_k,
+        )
+        anomaly_strategy = "fast_robust_random_projection"
+
     sampling = {
         **sampling,
         "reason": (
-            "Expensive neighbor/covariance detectors are bounded until the native "
-            "candidate-filtering anomaly engine is available."
-            if sampling["sampled"]
-            else "Full input fits within the anomaly execution budget."
+            "Research mode confirms anomalies with the heavier classical ensemble and "
+            "a bounded neural reconstruction detector."
+            if budget.mode == "research"
+            else "Standard/deep modes use vectorized robust and random-projection anomaly screening."
         ),
         "coverage": "sample" if sampling["sampled"] else "full",
+        "anomaly_strategy": anomaly_strategy,
+        "adaptive_strategy": anomaly_strategy,
+        "neural_reconstruction_enabled": budget.mode == "research",
     }
     return _attach_execution(
         payload,
@@ -125,13 +166,44 @@ def run_budgeted_anomalies(
     )
 
 
+def _attach_time_series_change_scan(
+    payload: dict[str, Any],
+    work: pd.DataFrame,
+) -> bool:
+    """Attach a cheap ordered mean-shift scan when time-series detection succeeded."""
+    if not payload.get("available"):
+        return False
+    date_column = payload.get("detected_date_column")
+    numeric_column = payload.get("numeric_column")
+    if not isinstance(date_column, str) or not isinstance(numeric_column, str):
+        return False
+    if date_column not in work.columns or numeric_column not in work.columns:
+        return False
+
+    parsed_dates = pd.to_datetime(work[date_column], errors="coerce", format="mixed")
+    ordered = pd.DataFrame({
+        "date": parsed_dates,
+        "value": pd.to_numeric(work[numeric_column], errors="coerce"),
+    }).dropna()
+    if ordered.empty:
+        return False
+    ordered = ordered.sort_values("date")
+    payload["mean_shift"] = scan_ordered_mean_shift(
+        ordered["value"],
+        windows=24,
+        threshold=8.0,
+        min_updates=8,
+    )
+    return True
+
+
 def run_budgeted_time_series(
     dataframe: pd.DataFrame,
     *,
     budget: ExecutionBudget,
     target_column: str | None = None,
 ) -> dict[str, Any]:
-    """Run time-series discovery/diagnostics on an order-preserving bounded frame."""
+    """Run bounded ordered time-series diagnostics plus change detection."""
     sample_limit = max(30, min(budget.time_series_sample_rows, max(len(dataframe), 1)))
     work, sampling = deterministic_sample_frame(
         dataframe,
@@ -139,15 +211,22 @@ def run_budgeted_time_series(
         preserve_order=True,
     )
     payload = detect_and_analyze_time_series(work, target_column=target_column)
+    change_detection_enabled = _attach_time_series_change_scan(payload, work)
     sampling = {
         **sampling,
         "reason": (
             "Time-series diagnostics use an order-preserving bounded view to avoid "
-            "unbounded date parsing, stationarity, PACF, STL, and forecasting work."
+            "unbounded date parsing, stationarity, PACF, STL, and forecasting work. "
+            "Detected numeric series also receive a bounded Page-Hinkley mean-shift scan."
             if sampling["sampled"]
-            else "Full input fits within the time-series execution budget."
+            else (
+                "Full input fits within the time-series execution budget; detected numeric "
+                "series also receive a bounded Page-Hinkley mean-shift scan."
+            )
         ),
         "temporal_order_preserved": True,
+        "mean_shift_detection_enabled": bool(change_detection_enabled),
+        "adaptive_strategy": "ordered_page_hinkley_mean_shift",
     }
     return _attach_execution(
         payload,
diff --git a/src/framevitals/deep_triage.py b/src/framevitals/deep_triage.py
new file mode 100644
index 0000000..8921581
--- /dev/null
+++ b/src/framevitals/deep_triage.py
@@ -0,0 +1,166 @@
+"""Cheap column triage for expensive deep-analysis routines.
+
+The profiler already describes every column. Deep statistical routines therefore
+should not blindly spend bootstrap/distribution-fitting work on every feature.
+This module scores all candidate columns with inexpensive vectorized statistics
+and returns a stable, bounded subset for heavyweight diagnostics.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+
+_MODE_LIMITS: dict[str, tuple[int, int]] = {
+    "quick": (4, 4),
+    "standard": (8, 6),
+    "deep": (12, 8),
+    "research": (40, 20),
+}
+
+
+@dataclass(frozen=True, slots=True)
+class DeepTriageResult:
+    selected_numeric: tuple[str, ...]
+    selected_categorical: tuple[str, ...]
+    numeric_scores: dict[str, float]
+    categorical_scores: dict[str, float]
+    numeric_available: int
+    categorical_available: int
+    numeric_limit: int
+    categorical_limit: int
+
+    @property
+    def selected_columns(self) -> tuple[str, ...]:
+        return self.selected_numeric + self.selected_categorical
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "method": "vectorized_interest_ranking",
+            "selected_numeric": list(self.selected_numeric),
+            "selected_categorical": list(self.selected_categorical),
+            "numeric_available": self.numeric_available,
+            "categorical_available": self.categorical_available,
+            "numeric_limit": self.numeric_limit,
+            "categorical_limit": self.categorical_limit,
+            "numeric_truncated": self.numeric_available > len(self.selected_numeric),
+            "categorical_truncated": self.categorical_available > len(self.selected_categorical),
+            "numeric_scores": dict(self.numeric_scores),
+            "categorical_scores": dict(self.categorical_scores),
+        }
+
+
+def _finite_score(value: Any) -> float:
+    try:
+        score = float(value)
+    except (TypeError, ValueError):
+        return 0.0
+    return score if np.isfinite(score) else 0.0
+
+
+def _rank_scores(scores: dict[str, float], original_order: list[str], limit: int) -> tuple[str, ...]:
+    order = {column: index for index, column in enumerate(original_order)}
+    ranked = sorted(
+        original_order,
+        key=lambda column: (-_finite_score(scores.get(column)), order[column]),
+    )
+    return tuple(ranked[: max(0, min(limit, len(ranked)))])
+
+
+def _numeric_interest_scores(frame: pd.DataFrame) -> dict[str, float]:
+    if frame.empty or frame.shape[1] == 0:
+        return {}
+
+    numeric = frame.replace([np.inf, -np.inf], np.nan)
+    n_rows = max(len(numeric), 1)
+    missing = numeric.isna().mean()
+    unique = numeric.nunique(dropna=True)
+
+    with np.errstate(all="ignore"):
+        skew = numeric.skew(axis=0, skipna=True).abs().fillna(0.0)
+        kurtosis = numeric.kurt(axis=0, skipna=True).abs().fillna(0.0)
+        std = numeric.std(axis=0, skipna=True).fillna(0.0)
+
+    scores: dict[str, float] = {}
+    for column in numeric.columns:
+        unique_count = int(unique.get(column, 0))
+        unique_ratio = unique_count / n_rows
+        near_constant = 1.0 if unique_count <= 2 or _finite_score(std.get(column)) <= 1e-12 else 0.0
+        score = (
+            min(_finite_score(skew.get(column)), 8.0) * 1.5
+            + min(_finite_score(kurtosis.get(column)), 20.0) * 0.65
+            + min(_finite_score(missing.get(column)), 1.0) * 5.0
+            + near_constant * 2.5
+            + min(unique_ratio, 1.0) * 0.35
+        )
+        scores[str(column)] = round(float(score), 6)
+    return scores
+
+
+def _categorical_interest_scores(frame: pd.DataFrame) -> dict[str, float]:
+    if frame.empty or frame.shape[1] == 0:
+        return {}
+
+    n_rows = max(len(frame), 1)
+    missing = frame.isna().mean()
+    unique = frame.nunique(dropna=True)
+    scores: dict[str, float] = {}
+
+    for column in frame.columns:
+        cardinality = int(unique.get(column, 0))
+        missing_rate = min(_finite_score(missing.get(column)), 1.0)
+        if cardinality <= 1:
+            relationship_value = 0.5
+        elif cardinality == 2:
+            relationship_value = 3.0
+        elif cardinality <= 12:
+            relationship_value = 2.5
+        elif cardinality <= 30:
+            relationship_value = 1.5
+        else:
+            relationship_value = 0.25
+        rarity_signal = min(cardinality / n_rows, 1.0) * 0.25
+        score = missing_rate * 8.0 + relationship_value + rarity_signal
+        scores[str(column)] = round(float(score), 6)
+    return scores
+
+
+def triage_deep_columns(dataframe: pd.DataFrame, *, mode: str) -> DeepTriageResult:
+    """Rank all deep-statistics candidates and return a bounded stable subset."""
+    if mode not in _MODE_LIMITS:
+        raise ValueError(f"Unknown analysis mode: {mode}")
+
+    numeric_columns = dataframe.select_dtypes(include=[np.number]).columns.tolist()
+    categorical_columns = dataframe.select_dtypes(
+        include=["object", "category", "bool", "string"],
+    ).columns.tolist()
+    numeric_limit, categorical_limit = _MODE_LIMITS[mode]
+
+    numeric_scores = _numeric_interest_scores(dataframe[numeric_columns]) if numeric_columns else {}
+    categorical_scores = (
+        _categorical_interest_scores(dataframe[categorical_columns])
+        if categorical_columns
+        else {}
+    )
+
+    selected_numeric = _rank_scores(numeric_scores, numeric_columns, numeric_limit)
+    selected_categorical = _rank_scores(
+        categorical_scores,
+        categorical_columns,
+        categorical_limit,
+    )
+
+    return DeepTriageResult(
+        selected_numeric=selected_numeric,
+        selected_categorical=selected_categorical,
+        numeric_scores=numeric_scores,
+        categorical_scores=categorical_scores,
+        numeric_available=len(numeric_columns),
+        categorical_available=len(categorical_columns),
+        numeric_limit=numeric_limit,
+        categorical_limit=categorical_limit,
+    )
diff --git a/src/framevitals/fast_anomaly.py b/src/framevitals/fast_anomaly.py
new file mode 100644
index 0000000..efa0896
--- /dev/null
+++ b/src/framevitals/fast_anomaly.py
@@ -0,0 +1,277 @@
+"""Fast bounded anomaly screening for standard and deep analysis modes.
+
+The scanner combines robust per-feature deviation, sparse random-projection tail
+scores, and projection-density rarity. It is fully vectorized and avoids tree,
+nearest-neighbour, and covariance fitting. Research mode can still run the
+heavier classical/neural ensemble for confirmation.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+
+def _to_unit(values: np.ndarray) -> np.ndarray:
+    values = np.asarray(values, dtype=np.float64)
+    finite = np.where(np.isfinite(values), values, np.nan)
+    if finite.size == 0 or np.all(np.isnan(finite)):
+        return np.zeros_like(values, dtype=np.float64)
+    lo = float(np.nanmin(finite))
+    hi = float(np.nanmax(finite))
+    if hi - lo <= 1e-12:
+        return np.zeros_like(values, dtype=np.float64)
+    out = (finite - lo) / (hi - lo)
+    return np.where(np.isfinite(out), out, 0.0)
+
+
+def _summary(scores: np.ndarray) -> dict[str, float]:
+    scores = np.asarray(scores, dtype=np.float64)
+    return {
+        "mean": round(float(np.mean(scores)), 4),
+        "median": round(float(np.median(scores)), 4),
+        "p95": round(float(np.quantile(scores, 0.95)), 4),
+        "p99": round(float(np.quantile(scores, 0.99)), 4),
+        "max": round(float(np.max(scores)), 4),
+    }
+
+
+def _prepare_numeric(
+    dataframe: pd.DataFrame,
+    *,
+    max_columns: int,
+) -> tuple[np.ndarray | None, list[str], dict[str, Any]]:
+    numeric = dataframe.select_dtypes(include=[np.number]).replace(
+        [np.inf, -np.inf],
+        np.nan,
+    )
+    original_columns = list(numeric.columns)
+    dropped_all_missing: list[str] = []
+    dropped_constant: list[str] = []
+
+    for column in list(numeric.columns):
+        valid = numeric[column].dropna()
+        if valid.empty:
+            numeric = numeric.drop(columns=[column])
+            dropped_all_missing.append(str(column))
+        elif valid.nunique() <= 1:
+            numeric = numeric.drop(columns=[column])
+            dropped_constant.append(str(column))
+
+    if numeric.empty:
+        return None, [], {
+            "numeric_columns_found": len(original_columns),
+            "used_columns": [],
+            "dropped_all_missing_columns": dropped_all_missing,
+            "dropped_constant_columns": dropped_constant,
+            "truncated_columns": False,
+        }
+
+    # Prefer columns with usable coverage and variation. This keeps ID-like or
+    # mostly-empty columns from crowding out informative dimensions.
+    variance = numeric.var(axis=0, skipna=True).fillna(0.0)
+    coverage = numeric.notna().mean(axis=0)
+    rank = (np.log1p(variance.abs()) + coverage).sort_values(ascending=False)
+    selected = rank.head(max_columns).index.tolist()
+    numeric = numeric[selected]
+
+    medians = numeric.median(axis=0, skipna=True)
+    numeric = numeric.fillna(medians).fillna(0.0)
+    matrix = numeric.to_numpy(dtype=np.float64)
+
+    center = np.median(matrix, axis=0)
+    mad = np.median(np.abs(matrix - center), axis=0)
+    std = np.std(matrix, axis=0, ddof=1)
+    scale = 1.4826 * mad
+    scale = np.where(scale > 1e-12, scale, np.where(std > 1e-12, std, 1.0))
+    standardized = (matrix - center) / scale
+
+    return standardized, [str(column) for column in selected], {
+        "numeric_columns_found": len(original_columns),
+        "used_columns": [str(column) for column in selected],
+        "dropped_all_missing_columns": dropped_all_missing,
+        "dropped_constant_columns": dropped_constant,
+        "truncated_columns": len(selected) < len(original_columns) - len(dropped_all_missing) - len(dropped_constant),
+        "scaling": "median_mad_with_std_fallback",
+    }
+
+
+def _projection_density_scores(projected: np.ndarray, bins: int = 32) -> np.ndarray:
+    n_rows, n_projections = projected.shape
+    if n_rows == 0 or n_projections == 0:
+        return np.zeros(n_rows, dtype=np.float64)
+    rarity = np.zeros((n_rows, n_projections), dtype=np.float64)
+
+    for index in range(n_projections):
+        values = projected[:, index]
+        counts, edges = np.histogram(values, bins=min(bins, max(4, int(np.sqrt(n_rows)))))
+        positions = np.searchsorted(edges, values, side="right") - 1
+        positions = np.clip(positions, 0, len(counts) - 1)
+        row_counts = counts[positions]
+        rarity[:, index] = -np.log((row_counts + 1.0) / (n_rows + len(counts)))
+
+    return _to_unit(np.mean(rarity, axis=1))
+
+
+def fast_anomaly_scan(
+    dataframe: pd.DataFrame,
+    *,
+    contamination: float = 0.05,
+    threshold: float = 0.6,
+    max_columns: int = 24,
+    projections: int = 12,
+    top_k: int = 25,
+    random_state: int = 42,
+) -> dict[str, Any]:
+    """Return a fast explainable anomaly screen over a bounded dataframe."""
+    if max_columns < 1:
+        raise ValueError("max_columns must be at least 1.")
+    if projections < 2:
+        raise ValueError("projections must be at least 2.")
+    if top_k < 1:
+        raise ValueError("top_k must be at least 1.")
+    if not 0.0 <= threshold <= 1.0:
+        raise ValueError("threshold must be between 0 and 1.")
+    contamination = float(np.clip(contamination, 0.001, 0.5))
+
+    matrix, used_columns, preparation = _prepare_numeric(
+        dataframe,
+        max_columns=max_columns,
+    )
+    if matrix is None or len(matrix) < 20:
+        return {
+            "available": False,
+            "message": "Need at least 20 rows and one non-constant numeric column.",
+            "preparation": preparation,
+        }
+
+    n_rows, n_features = matrix.shape
+    robust_abs = np.abs(matrix)
+    top_feature_count = min(3, n_features)
+    if top_feature_count == n_features:
+        robust_raw = robust_abs.mean(axis=1)
+    else:
+        top = np.partition(robust_abs, n_features - top_feature_count, axis=1)[
+            :, -top_feature_count:
+        ]
+        robust_raw = top.mean(axis=1)
+    robust_score = _to_unit(robust_raw)
+
+    rng = np.random.default_rng(random_state)
+    projection_matrix = rng.choice(
+        np.array([-1.0, 0.0, 1.0]),
+        size=(n_features, projections),
+        p=[0.25, 0.50, 0.25],
+    )
+    for index in range(projections):
+        nonzero = np.count_nonzero(projection_matrix[:, index])
+        if nonzero == 0:
+            projection_matrix[index % n_features, index] = 1.0
+            nonzero = 1
+        projection_matrix[:, index] /= math.sqrt(nonzero)
+
+    projected = matrix @ projection_matrix
+    projection_center = np.median(projected, axis=0)
+    projection_mad = np.median(np.abs(projected - projection_center), axis=0)
+    projection_scale = np.where(1.4826 * projection_mad > 1e-12, 1.4826 * projection_mad, 1.0)
+    projection_z = np.abs((projected - projection_center) / projection_scale)
+    projection_top = np.partition(
+        projection_z,
+        max(0, projections - min(3, projections)),
+        axis=1,
+    )[:, -min(3, projections):]
+    projection_tail_score = _to_unit(projection_top.mean(axis=1))
+    density_score = _projection_density_scores(projected)
+
+    detector_scores = {
+        "robust_feature_deviation": robust_score,
+        "random_projection_tail": projection_tail_score,
+        "random_projection_density": density_score,
+    }
+    ensemble = (
+        0.50 * robust_score
+        + 0.35 * projection_tail_score
+        + 0.15 * density_score
+    )
+    ensemble = _to_unit(ensemble)
+
+    detector_names = list(detector_scores)
+    vote_thresholds = {
+        name: float(np.quantile(scores, 1.0 - contamination))
+        for name, scores in detector_scores.items()
+    }
+    votes = np.column_stack([
+        detector_scores[name] >= vote_thresholds[name]
+        for name in detector_names
+    ])
+    agreement_count = votes.sum(axis=1)
+    majority_required = max(1, math.ceil(len(detector_names) / 2))
+    consensus_mask = agreement_count >= majority_required
+    flagged_mask = ensemble >= threshold
+
+    top_indices = np.argsort(ensemble)[::-1][: min(top_k, n_rows)]
+    top_rows: list[dict[str, Any]] = []
+    for position in top_indices:
+        feature_order = np.argsort(robust_abs[position])[::-1][: min(3, n_features)]
+        top_rows.append({
+            "row_index": (
+                int(dataframe.index[position])
+                if isinstance(dataframe.index[position], (int, np.integer))
+                else str(dataframe.index[position])
+            ),
+            "robust_feature_deviation": round(float(robust_score[position]), 4),
+            "random_projection_tail": round(float(projection_tail_score[position]), 4),
+            "random_projection_density": round(float(density_score[position]), 4),
+            "ensemble": round(float(ensemble[position]), 4),
+            "flagged": bool(flagged_mask[position]),
+            "agreement_count": int(agreement_count[position]),
+            "agreement_fraction": round(float(agreement_count[position] / len(detector_names)), 4),
+            "top_feature_deviations": [
+                {
+                    "feature": used_columns[int(feature_index)],
+                    "standardized_deviation": round(float(robust_abs[position, feature_index]), 4),
+                    "direction": "high" if matrix[position, feature_index] >= 0 else "low",
+                }
+                for feature_index in feature_order
+            ],
+        })
+
+    flagged_count = int(flagged_mask.sum())
+    consensus_count = int(consensus_mask.sum())
+    return {
+        "available": True,
+        "method": "fast_robust_random_projection",
+        "n_rows_scored": int(n_rows),
+        "used_columns": used_columns,
+        "preparation": preparation,
+        "detectors_run": detector_names,
+        "detectors_failed": {},
+        "detectors_skipped": {},
+        "detector_summaries": {
+            name: _summary(scores) for name, scores in detector_scores.items()
+        },
+        "detector_vote_thresholds": {
+            name: round(value, 4) for name, value in vote_thresholds.items()
+        },
+        "threshold": float(threshold),
+        "contamination": contamination,
+        "expected_anomaly_count": int(math.ceil(n_rows * contamination)),
+        "flagged_count": flagged_count,
+        "flagged_fraction": round(float(flagged_count / n_rows), 4),
+        "consensus": {
+            "majority_detectors_required": majority_required,
+            "flagged_count": consensus_count,
+            "flagged_fraction": round(float(consensus_count / n_rows), 4),
+        },
+        "ensemble_summary": _summary(ensemble),
+        "top_rows": top_rows,
+        "projection_count": int(projections),
+        "interpretation": (
+            "Standard/deep anomaly screening combines robust feature deviations, sparse "
+            "random-projection tail behaviour, and projection-density rarity. Research mode "
+            "can confirm candidates with the heavier classical and neural ensembles."
+        ),
+    }
diff --git a/src/framevitals/neural_anomaly.py b/src/framevitals/neural_anomaly.py
new file mode 100644
index 0000000..85bde9c
--- /dev/null
+++ b/src/framevitals/neural_anomaly.py
@@ -0,0 +1,138 @@
+"""Small neural reconstruction detector for research-mode anomaly analysis.
+
+This intentionally uses the existing scikit-learn dependency instead of adding a
+large deep-learning runtime. The network is bounded by rows, columns and epochs
+and is designed as an additional nonlinear anomaly view, not as a replacement
+for deterministic statistical checks.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from sklearn.impute import SimpleImputer
+from sklearn.neural_network import MLPRegressor
+from sklearn.preprocessing import StandardScaler
+
+
+def _unit_scale(values: np.ndarray) -> np.ndarray:
+    values = np.asarray(values, dtype=float)
+    if values.size == 0:
+        return values
+    lo = float(np.min(values))
+    hi = float(np.max(values))
+    if not np.isfinite(lo) or not np.isfinite(hi) or hi - lo <= 1e-12:
+        return np.zeros_like(values)
+    return (values - lo) / (hi - lo)
+
+
+def neural_reconstruction_anomalies(
+    dataframe: pd.DataFrame,
+    *,
+    max_rows: int = 3_000,
+    max_columns: int = 24,
+    max_iter: int = 35,
+    top_k: int = 25,
+    random_state: int = 42,
+) -> dict[str, Any]:
+    """Score rows with a tiny autoencoder-like MLP reconstruction network."""
+    if max_rows < 20:
+        raise ValueError("max_rows must be at least 20.")
+    if max_columns < 1:
+        raise ValueError("max_columns must be at least 1.")
+    if max_iter < 1:
+        raise ValueError("max_iter must be positive.")
+    if top_k < 1:
+        raise ValueError("top_k must be positive.")
+
+    numeric = dataframe.select_dtypes(include=[np.number]).replace(
+        [np.inf, -np.inf],
+        np.nan,
+    )
+    dropped_constant: list[str] = []
+    for column in list(numeric.columns):
+        if numeric[column].nunique(dropna=True) <= 1:
+            numeric = numeric.drop(columns=[column])
+            dropped_constant.append(str(column))
+
+    if numeric.shape[1] < 2 or len(numeric) < 20:
+        return {
+            "available": False,
+            "reason": "Need at least 20 rows and two non-constant numeric columns.",
+            "dropped_constant_columns": dropped_constant,
+        }
+
+    variances = numeric.var(axis=0, skipna=True).fillna(0.0)
+    selected = variances.sort_values(ascending=False).head(max_columns).index.tolist()
+    numeric = numeric[selected]
+
+    source_rows = len(numeric)
+    if source_rows > max_rows:
+        positions = np.linspace(0, source_rows - 1, num=max_rows, dtype=np.int64)
+        positions = np.unique(positions)
+        work = numeric.iloc[positions]
+        sampled = True
+    else:
+        work = numeric
+        sampled = False
+
+    imputer = SimpleImputer(strategy="median")
+    scaler = StandardScaler()
+    matrix = imputer.fit_transform(work)
+    matrix = scaler.fit_transform(matrix)
+
+    n_features = matrix.shape[1]
+    bottleneck = max(2, min(12, n_features // 3 if n_features >= 6 else n_features - 1))
+    hidden = max(bottleneck + 1, min(24, max(4, n_features // 2)))
+    network = MLPRegressor(
+        hidden_layer_sizes=(hidden, bottleneck, hidden),
+        activation="relu",
+        solver="adam",
+        alpha=1e-4,
+        batch_size=min(256, max(16, len(matrix) // 10)),
+        learning_rate_init=1e-3,
+        max_iter=max_iter,
+        early_stopping=True,
+        validation_fraction=0.1,
+        n_iter_no_change=5,
+        random_state=random_state,
+    )
+    network.fit(matrix, matrix)
+    reconstructed = network.predict(matrix)
+    errors = np.mean((matrix - reconstructed) ** 2, axis=1)
+    scores = _unit_scale(errors)
+
+    order = np.argsort(scores)[::-1][: min(top_k, len(scores))]
+    top_rows = [
+        {
+            "row_index": str(work.index[int(index)]),
+            "score": round(float(scores[int(index)]), 6),
+            "reconstruction_error": round(float(errors[int(index)]), 6),
+        }
+        for index in order
+    ]
+
+    return {
+        "available": True,
+        "method": "bounded_mlp_reconstruction",
+        "source_rows": int(source_rows),
+        "sample_rows": int(len(work)),
+        "sampled": sampled,
+        "used_columns": [str(column) for column in selected],
+        "columns_available": int(dataframe.select_dtypes(include=[np.number]).shape[1]),
+        "columns_used": int(len(selected)),
+        "truncated_columns": bool(len(selected) < dataframe.select_dtypes(include=[np.number]).shape[1]),
+        "architecture": [int(n_features), int(hidden), int(bottleneck), int(hidden), int(n_features)],
+        "iterations": int(network.n_iter_),
+        "loss": round(float(network.loss_), 6),
+        "score_summary": {
+            "mean": round(float(np.mean(scores)), 6),
+            "p95": round(float(np.quantile(scores, 0.95)), 6),
+            "p99": round(float(np.quantile(scores, 0.99)), 6),
+            "max": round(float(np.max(scores)), 6),
+        },
+        "top_rows": top_rows,
+        "dropped_constant_columns": dropped_constant,
+    }
diff --git a/src/framevitals/stream_change.py b/src/framevitals/stream_change.py
new file mode 100644
index 0000000..ef1ad36
--- /dev/null
+++ b/src/framevitals/stream_change.py
@@ -0,0 +1,174 @@
+"""Low-overhead change detection for streaming and ordered numeric data.
+
+The detector consumes aggregate observations rather than every raw cell. That
+keeps Python overhead proportional to ``columns * windows`` while still
+surfacing sustained mean shifts in wide data and ordered time series.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import math
+
+import numpy as np
+import pandas as pd
+
+
+@dataclass(slots=True)
+class PageHinkleyMeanShift:
+    """Scale-adaptive Page-Hinkley/CUSUM detector for sequential batch means."""
+
+    threshold: float = 8.0
+    delta: float = 0.5
+    min_updates: int = 8
+    alpha: float = 0.995
+    count: int = 0
+    mean: float = 0.0
+    m2: float = 0.0
+    cumulative_up: float = 0.0
+    cumulative_down: float = 0.0
+    max_score: float = 0.0
+    detected: bool = False
+    direction: str | None = None
+    detected_at: int | None = None
+
+    def __post_init__(self) -> None:
+        if self.threshold <= 0:
+            raise ValueError("threshold must be positive.")
+        if self.delta < 0:
+            raise ValueError("delta must be non-negative.")
+        if self.min_updates < 3:
+            raise ValueError("min_updates must be at least 3.")
+        if not 0 < self.alpha <= 1:
+            raise ValueError("alpha must be in (0, 1].")
+
+    @property
+    def variance(self) -> float | None:
+        if self.count < 2:
+            return None
+        return self.m2 / (self.count - 1)
+
+    @property
+    def std(self) -> float | None:
+        variance = self.variance
+        if variance is None or variance <= 0:
+            return None
+        return math.sqrt(variance)
+
+    def update(self, value: float | int | None) -> bool:
+        """Update with the next aggregate mean and return current detection state."""
+        if value is None:
+            return self.detected
+        x = float(value)
+        if not math.isfinite(x):
+            return self.detected
+
+        previous_mean = self.mean
+        previous_std = self.std
+        self.count += 1
+
+        if self.count == 1:
+            self.mean = x
+            return self.detected
+
+        difference = x - self.mean
+        self.mean += difference / self.count
+        self.m2 += difference * (x - self.mean)
+
+        if self.count < self.min_updates:
+            return self.detected
+
+        scale = previous_std if previous_std is not None and previous_std > 1e-12 else None
+        if scale is None:
+            scale = max(abs(previous_mean) * 1e-6, 1e-9)
+
+        # With standardized residuals, a 0.5-sigma allowance gives a much more
+        # useful false-positive tradeoff than a tiny raw-unit delta. The one-sided
+        # reset also prevents stationary random walks from accumulating forever.
+        residual = float(np.clip((x - previous_mean) / scale, -8.0, 8.0))
+        self.cumulative_up = max(
+            0.0,
+            self.alpha * self.cumulative_up + residual - self.delta,
+        )
+        self.cumulative_down = max(
+            0.0,
+            self.alpha * self.cumulative_down - residual - self.delta,
+        )
+
+        score = max(self.cumulative_up, self.cumulative_down)
+        self.max_score = max(self.max_score, float(score))
+
+        if not self.detected and score >= self.threshold:
+            self.detected = True
+            self.direction = "up" if self.cumulative_up >= self.cumulative_down else "down"
+            self.detected_at = self.count
+        return self.detected
+
+    def snapshot(self) -> dict[str, Any]:
+        return {
+            "method": "page_hinkley_batch_means",
+            "updates": int(self.count),
+            "detected": bool(self.detected),
+            "direction": self.direction,
+            "detected_at_batch": self.detected_at,
+            "max_score": round(float(self.max_score), 6),
+            "threshold": float(self.threshold),
+            "delta": float(self.delta),
+            "minimum_batches": int(self.min_updates),
+            "sufficient_batches": bool(self.count >= self.min_updates),
+        }
+
+
+def scan_ordered_mean_shift(
+    series: pd.Series,
+    *,
+    windows: int = 24,
+    threshold: float = 8.0,
+    min_updates: int = 8,
+) -> dict[str, Any]:
+    """Detect sustained mean changes using bounded contiguous window summaries."""
+    if windows < min_updates:
+        raise ValueError("windows must be at least min_updates.")
+
+    numeric = pd.to_numeric(series, errors="coerce").replace([np.inf, -np.inf], np.nan)
+    values = numeric.to_numpy(dtype=np.float64, na_value=np.nan)
+    if values.size < min_updates * 4:
+        return {
+            "available": False,
+            "reason": "Too few ordered observations for bounded change detection.",
+            "method": "page_hinkley_window_means",
+            "observations": int(values.size),
+        }
+
+    effective_windows = min(int(windows), max(min_updates, values.size // 4))
+    chunks = np.array_split(values, effective_windows)
+    detector = PageHinkleyMeanShift(
+        threshold=threshold,
+        min_updates=min_updates,
+    )
+    window_means: list[float | None] = []
+    for chunk in chunks:
+        finite = chunk[np.isfinite(chunk)]
+        mean = float(np.mean(finite)) if finite.size else None
+        window_means.append(mean)
+        detector.update(mean)
+
+    snapshot = detector.snapshot()
+    valid_means = [value for value in window_means if value is not None]
+    return {
+        "available": bool(valid_means),
+        "method": "page_hinkley_window_means",
+        "observations": int(values.size),
+        "windows": int(effective_windows),
+        "window_means_preview": [
+            None if value is None else round(float(value), 6)
+            for value in window_means[-12:]
+        ],
+        "detected": bool(snapshot["detected"]),
+        "direction": snapshot["direction"],
+        "detected_at_window": snapshot["detected_at_batch"],
+        "max_score": snapshot["max_score"],
+        "threshold": snapshot["threshold"],
+    }
diff --git a/src/framevitals/streaming_profile.py b/src/framevitals/streaming_profile.py
index fd6f9b4..4249f0a 100644
--- a/src/framevitals/streaming_profile.py
+++ b/src/framevitals/streaming_profile.py
@@ -24,6 +24,11 @@
 )
 from framevitals.profiler import _bounded_correlations, series_to_dict
 from framevitals.sources import StreamingDatasetSource
+from framevitals.streaming_sketches import (
+    NumpyLogQuantileSketch,
+    PYTHON_NUMERIC_SKETCH_CELL_BUDGET,
+    should_use_full_stream_numpy_sketch,
+)
 
 
 STREAM_BATCH_SIZE = 65_536
@@ -204,19 +209,32 @@ def _summary_from_native_snapshot(payload: dict[str, Any]) -> dict[str, Any]:
 def _summary_from_python_state(
     state: NumericColumnState,
     sample: pd.Series,
+    quantile_sketch: NumpyLogQuantileSketch | None = None,
 ) -> dict[str, Any]:
-    finite_sample = pd.to_numeric(sample, errors="coerce")
-    finite_values = finite_sample.to_numpy(dtype="float64", na_value=np.nan)
-    finite_sample = finite_sample[np.isfinite(finite_values)]
-    quantiles = finite_sample.quantile([0.25, 0.5, 0.75]) if len(finite_sample) else pd.Series()
+    if quantile_sketch is not None:
+        q25 = quantile_sketch.quantile(0.25)
+        q50 = quantile_sketch.quantile(0.50)
+        q75 = quantile_sketch.quantile(0.75)
+    else:
+        finite_sample = pd.to_numeric(sample, errors="coerce")
+        finite_values = finite_sample.to_numpy(dtype="float64", na_value=np.nan)
+        finite_sample = finite_sample[np.isfinite(finite_values)]
+        quantiles = (
+            finite_sample.quantile([0.25, 0.5, 0.75])
+            if len(finite_sample)
+            else pd.Series()
+        )
+        q25 = quantiles.get(0.25)
+        q50 = quantiles.get(0.5)
+        q75 = quantiles.get(0.75)
     return {
         "count": state.count,
         "mean": _round_optional(state.mean if state.count else None),
         "std": _round_optional(state.std),
         "min": _round_optional(state.minimum),
-        "25%": _round_optional(quantiles.get(0.25)),
-        "50%": _round_optional(quantiles.get(0.5)),
-        "75%": _round_optional(quantiles.get(0.75)),
+        "25%": _round_optional(q25),
+        "50%": _round_optional(q50),
+        "75%": _round_optional(q75),
         "max": _round_optional(state.maximum),
     }
 
@@ -381,10 +399,19 @@ def build_streaming_profile(
     preview_frame: pd.DataFrame | None = None
     missing_counts = {column: 0 for column in columns}
     selected_backend = resolve_numeric_backend()
+    use_numpy_quantile_sketches = (
+        selected_backend == "numpy"
+        and should_use_full_stream_numpy_sketch(int(rows), len(numeric_cols))
+    )
 
     native_accumulators: dict[str, Any] = {}
     native_string_accumulators: dict[str, Any] = {}
     python_states = {column: NumericColumnState() for column in numeric_cols}
+    numpy_quantile_sketches = (
+        {column: NumpyLogQuantileSketch() for column in numeric_cols}
+        if use_numpy_quantile_sketches
+        else {}
+    )
     if selected_backend == "rust":
         native_accumulators = {
             column: create_numeric_accumulator(stream_id=index)
@@ -425,6 +452,9 @@ def build_streaming_profile(
                 python_states[column] = python_states[column].merge(
                     _state_from_payload(numeric_payload)
                 )
+                quantile_sketch = numpy_quantile_sketches.get(column)
+                if quantile_sketch is not None:
+                    quantile_sketch.update(values)
 
         offset += int(batch.num_rows)
 
@@ -457,6 +487,7 @@ def build_streaming_profile(
             "method": "native_streaming_accumulator",
             "approximate_quantiles": True,
             "quantile_relative_accuracy": quantile_accuracy,
+            "quantile_source": "full_stream_sketch",
             "finite_only_moments": True,
             "columns_profiled": len(numeric_cols),
             "raw_observations_retained": False,
@@ -467,17 +498,34 @@ def build_streaming_profile(
             numeric_summary[column] = _summary_from_python_state(
                 state,
                 sample[column] if column in sample else pd.Series(dtype="float64"),
+                numpy_quantile_sketches.get(column),
             )
             missing_counts[column] = state.missing
-        numeric_metadata = {
-            "backend": "numpy",
-            "method": "mergeable_streaming_moments_with_row_sample_quantiles",
-            "approximate_quantiles": len(sample) < rows,
-            "quantile_sample_rows": int(len(sample)),
-            "finite_only_moments": True,
-            "columns_profiled": len(numeric_cols),
-            "raw_observations_retained": False,
-        }
+        if use_numpy_quantile_sketches:
+            numeric_metadata = {
+                "backend": "numpy",
+                "method": "mergeable_streaming_moments_with_full_stream_log_quantiles",
+                "approximate_quantiles": True,
+                "quantile_relative_accuracy": 0.01,
+                "quantile_source": "full_stream_sketch",
+                "quantile_cell_budget": int(PYTHON_NUMERIC_SKETCH_CELL_BUDGET),
+                "finite_only_moments": True,
+                "columns_profiled": len(numeric_cols),
+                "raw_observations_retained": False,
+            }
+        else:
+            numeric_metadata = {
+                "backend": "numpy",
+                "method": "mergeable_streaming_moments_with_row_sample_quantiles",
+                "approximate_quantiles": len(sample) < rows,
+                "quantile_sample_rows": int(len(sample)),
+                "quantile_source": "bounded_row_sample",
+                "quantile_cell_budget": int(PYTHON_NUMERIC_SKETCH_CELL_BUDGET),
+                "quantile_sketch_skipped_for_cost": bool(numeric_cols),
+                "finite_only_moments": True,
+                "columns_profiled": len(numeric_cols),
+                "raw_observations_retained": False,
+            }
 
     categorical_summary: dict[str, dict[str, Any]] = {}
     native_categorical_columns: list[str] = []
@@ -594,9 +642,11 @@ def build_streaming_profile(
             ),
             "sample_strategy": "evenly_spaced_global_rows",
             "numeric_backend": selected_backend,
+            "numpy_full_stream_quantile_sketches": bool(use_numpy_quantile_sketches),
+            "numpy_numeric_sketch_cell_budget": int(PYTHON_NUMERIC_SKETCH_CELL_BUDGET),
             "native_string_sketch_columns": native_categorical_columns,
         },
     }
     if return_sample:
         return payload, sample
-    return payload
\ No newline at end of file
+    return payload
diff --git a/src/framevitals/streaming_sketches.py b/src/framevitals/streaming_sketches.py
new file mode 100644
index 0000000..36f8857
--- /dev/null
+++ b/src/framevitals/streaming_sketches.py
@@ -0,0 +1,155 @@
+"""Pure-NumPy bounded-memory sketches for non-native streaming execution.
+
+The Rust backend already owns the fastest full-stream sketch implementation.
+These fallbacks mirror the same logarithmic-quantile semantics for datasets
+small enough that Python/NumPy sketch maintenance is cheaper than retaining a
+large row sample. Ultra-wide inputs deliberately keep the existing sampled
+fallback so this module never turns a memory fix into a CPU regression.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
+
+
+PYTHON_NUMERIC_SKETCH_CELL_BUDGET = 50_000_000
+DEFAULT_RELATIVE_ACCURACY = 0.01
+DEFAULT_ZERO_THRESHOLD = 1.0e-12
+
+
+def should_use_full_stream_numpy_sketch(
+    rows: int,
+    numeric_columns: int,
+    *,
+    cell_budget: int = PYTHON_NUMERIC_SKETCH_CELL_BUDGET,
+) -> bool:
+    """Return whether NumPy sketch work fits the configured cell-cost budget."""
+    if rows < 0 or numeric_columns < 0:
+        raise ValueError("rows and numeric_columns must be non-negative.")
+    if cell_budget < 1:
+        raise ValueError("cell_budget must be positive.")
+    return int(rows) * int(numeric_columns) <= int(cell_budget)
+
+
+@dataclass(slots=True)
+class NumpyLogQuantileSketch:
+    """Mergeable relative-accuracy logarithmic quantile sketch.
+
+    This mirrors FrameVitals' native ``LogQuantileSketch`` rather than retaining
+    raw observations. Batch updates are vectorized with NumPy and only unique
+    logarithmic bins are folded into Python dictionaries.
+    """
+
+    relative_accuracy: float = DEFAULT_RELATIVE_ACCURACY
+    zero_threshold: float = DEFAULT_ZERO_THRESHOLD
+    negative: dict[int, int] = field(default_factory=dict)
+    positive: dict[int, int] = field(default_factory=dict)
+    zero_count: int = 0
+    count: int = 0
+
+    def __post_init__(self) -> None:
+        if not 0.0 < float(self.relative_accuracy) < 1.0:
+            raise ValueError("relative_accuracy must be between 0 and 1.")
+        if self.zero_threshold < 0:
+            raise ValueError("zero_threshold must be non-negative.")
+
+    @property
+    def _log_gamma(self) -> float:
+        gamma = (1.0 + self.relative_accuracy) / (1.0 - self.relative_accuracy)
+        return float(np.log(gamma))
+
+    def _update_side(self, target: dict[int, int], magnitudes: np.ndarray) -> None:
+        if magnitudes.size == 0:
+            return
+        keys = np.floor(np.log(magnitudes) / self._log_gamma).astype(np.int64)
+        unique, counts = np.unique(keys, return_counts=True)
+        for key, amount in zip(unique.tolist(), counts.tolist(), strict=True):
+            integer_key = int(key)
+            target[integer_key] = target.get(integer_key, 0) + int(amount)
+
+    def update(self, values: Any) -> "NumpyLogQuantileSketch":
+        array = np.asarray(values, dtype=np.float64).reshape(-1)
+        finite = array[np.isfinite(array)]
+        if finite.size == 0:
+            return self
+
+        self.count += int(finite.size)
+        absolute = np.abs(finite)
+        zero_mask = absolute <= self.zero_threshold
+        self.zero_count += int(np.count_nonzero(zero_mask))
+
+        nonzero = finite[~zero_mask]
+        if nonzero.size:
+            negative = nonzero[nonzero < 0.0]
+            positive = nonzero[nonzero > 0.0]
+            self._update_side(self.negative, -negative)
+            self._update_side(self.positive, positive)
+        return self
+
+    def merge(self, other: "NumpyLogQuantileSketch") -> "NumpyLogQuantileSketch":
+        if not np.isclose(self.relative_accuracy, other.relative_accuracy):
+            raise ValueError("quantile sketch accuracy mismatch")
+        if not np.isclose(self.zero_threshold, other.zero_threshold):
+            raise ValueError("quantile sketch zero-threshold mismatch")
+        self.count += other.count
+        self.zero_count += other.zero_count
+        for key, amount in other.negative.items():
+            self.negative[key] = self.negative.get(key, 0) + amount
+        for key, amount in other.positive.items():
+            self.positive[key] = self.positive.get(key, 0) + amount
+        return self
+
+    def _representative(self, key: int) -> float:
+        return float(np.exp((float(key) + 0.5) * self._log_gamma))
+
+    def quantile(self, q: float) -> float | None:
+        if self.count == 0:
+            return None
+        if not 0.0 <= float(q) <= 1.0:
+            raise ValueError("q must be between 0 and 1.")
+
+        target = int(np.floor(float(q) * (self.count - 1)))
+        seen = 0
+        for key in sorted(self.negative, reverse=True):
+            amount = self.negative[key]
+            if target < seen + amount:
+                return -self._representative(key)
+            seen += amount
+
+        if target < seen + self.zero_count:
+            return 0.0
+        seen += self.zero_count
+
+        for key in sorted(self.positive):
+            amount = self.positive[key]
+            if target < seen + amount:
+                return self._representative(key)
+            seen += amount
+
+        if self.positive:
+            return self._representative(max(self.positive))
+        if self.negative:
+            return -self._representative(min(self.negative))
+        return 0.0
+
+    @property
+    def bin_count(self) -> int:
+        return len(self.negative) + len(self.positive) + int(self.zero_count > 0)
+
+    def snapshot(self) -> dict[str, Any]:
+        return {
+            "method": "numpy_log_quantile_sketch",
+            "count": int(self.count),
+            "relative_accuracy": float(self.relative_accuracy),
+            "bin_count": int(self.bin_count),
+            "p01": self.quantile(0.01),
+            "p05": self.quantile(0.05),
+            "p25": self.quantile(0.25),
+            "p50": self.quantile(0.50),
+            "p75": self.quantile(0.75),
+            "p95": self.quantile(0.95),
+            "p99": self.quantile(0.99),
+        }
diff --git a/tests/test_adaptive_execution.py b/tests/test_adaptive_execution.py
index 4c98fc8..d3bad1d 100644
--- a/tests/test_adaptive_execution.py
+++ b/tests/test_adaptive_execution.py
@@ -106,7 +106,7 @@ def fake_anomalies(frame, **kwargs):
         return {"available": True, "n_rows_scored": len(frame)}
 
     monkeypatch.setattr(
-        "framevitals.budgeted_analysis.detect_anomalies_ensemble",
+        "framevitals.budgeted_analysis.fast_anomaly_scan",
         fake_anomalies,
     )
     frame = pd.DataFrame({"x": np.arange(20_000)})
@@ -117,6 +117,36 @@ def fake_anomalies(frame, **kwargs):
     assert seen["rows"] == budget.anomaly_sample_rows
     assert result["execution"]["coverage"] == "sample"
     assert result["execution"]["sample_rows"] == budget.anomaly_sample_rows
+    assert result["execution"]["anomaly_strategy"] == "fast_robust_random_projection"
+
+
+def test_research_anomaly_adapter_keeps_heavy_confirmation(monkeypatch):
+    seen = {"classical": 0, "neural": 0}
+
+    def fake_classical(frame, **kwargs):
+        seen["classical"] += 1
+        return {"available": True}
+
+    def fake_neural(frame, **kwargs):
+        seen["neural"] += 1
+        return {"available": True}
+
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.detect_anomalies_ensemble",
+        fake_classical,
+    )
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.neural_reconstruction_anomalies",
+        fake_neural,
+    )
+    frame = pd.DataFrame({"x": np.arange(200), "y": np.arange(200) * 2})
+    budget = derive_execution_budget(len(frame), 2, mode="research")
+
+    result = run_budgeted_anomalies(frame, budget=budget)
+
+    assert seen == {"classical": 1, "neural": 1}
+    assert result["execution"]["anomaly_strategy"] == "classical_ensemble_plus_neural_reconstruction"
+    assert result["execution"]["neural_reconstruction_enabled"] is True
 
 
 def test_time_series_adapter_preserves_order(monkeypatch):
diff --git a/tests/test_arrow_memory.py b/tests/test_arrow_memory.py
index 9de9bdd..ceed2dc 100644
--- a/tests/test_arrow_memory.py
+++ b/tests/test_arrow_memory.py
@@ -68,6 +68,37 @@ def fail_load(self):
     assert result["missing_counts"]["value"] == int(np.isnan(table["value"].to_numpy()).sum())
 
 
+def test_numpy_streaming_profile_uses_full_stream_quantile_sketch_when_affordable(monkeypatch):
+    table = _table(12_000)
+    monkeypatch.setattr("framevitals.streaming_profile.resolve_numeric_backend", lambda: "numpy")
+
+    result = framevitals.profile(table)
+    numeric = result["numeric_summary_metadata"]
+    streaming = result["streaming_metadata"]
+
+    assert numeric["backend"] == "numpy"
+    assert numeric["quantile_source"] == "full_stream_sketch"
+    assert numeric["quantile_relative_accuracy"] == 0.01
+    assert streaming["numpy_full_stream_quantile_sketches"] is True
+    assert result["numeric_summary"]["other"]["50%"] == pytest.approx(11_999, rel=0.03)
+
+
+def test_numpy_streaming_profile_can_fall_back_to_bounded_sample_quantiles(monkeypatch):
+    table = _table(12_000)
+    monkeypatch.setattr("framevitals.streaming_profile.resolve_numeric_backend", lambda: "numpy")
+    monkeypatch.setattr(
+        "framevitals.streaming_profile.should_use_full_stream_numpy_sketch",
+        lambda rows, columns: False,
+    )
+
+    result = framevitals.profile(table)
+    numeric = result["numeric_summary_metadata"]
+
+    assert numeric["quantile_source"] == "bounded_row_sample"
+    assert numeric["quantile_sketch_skipped_for_cost"] is True
+    assert result["streaming_metadata"]["numpy_full_stream_quantile_sketches"] is False
+
+
 def test_public_analyze_dispatches_arrow_table_through_streaming_pipeline(monkeypatch):
     table = _table(6_000)
 
diff --git a/tests/test_deep_triage.py b/tests/test_deep_triage.py
new file mode 100644
index 0000000..d366e06
--- /dev/null
+++ b/tests/test_deep_triage.py
@@ -0,0 +1,70 @@
+import numpy as np
+import pandas as pd
+
+from framevitals.budgeted_analysis import run_budgeted_deep_statistics
+from framevitals.deep_triage import triage_deep_columns
+from framevitals.execution import derive_execution_budget
+
+
+def _wide_frame(rows: int = 2_000) -> pd.DataFrame:
+    rng = np.random.default_rng(42)
+    payload: dict[str, object] = {}
+    for index in range(30):
+        payload[f"n{index}"] = rng.normal(size=rows)
+    payload["n27"] = rng.lognormal(mean=0.0, sigma=2.0, size=rows)
+    payload["n28"] = np.where(np.arange(rows) % 4 == 0, np.nan, rng.normal(size=rows))
+    payload["n29"] = np.ones(rows)
+    for index in range(15):
+        payload[f"c{index}"] = np.array([f"g-{value % (index + 2)}" for value in range(rows)], dtype=object)
+    payload["c14"] = np.where(np.arange(rows) % 3 == 0, None, "rare")
+    return pd.DataFrame(payload)
+
+
+def test_deep_triage_bounds_columns_and_prioritizes_diagnostic_signals():
+    frame = _wide_frame()
+    result = triage_deep_columns(frame, mode="deep")
+
+    assert len(result.selected_numeric) == 12
+    assert len(result.selected_categorical) == 8
+    assert {"n27", "n28", "n29"} <= set(result.selected_numeric)
+    assert "c14" in result.selected_categorical
+    metadata = result.to_dict()
+    assert metadata["numeric_truncated"] is True
+    assert metadata["categorical_truncated"] is True
+
+
+def test_research_mode_keeps_a_larger_deep_diagnostic_surface():
+    frame = _wide_frame()
+    deep = triage_deep_columns(frame, mode="deep")
+    research = triage_deep_columns(frame, mode="research")
+
+    assert len(research.selected_numeric) > len(deep.selected_numeric)
+    assert len(research.selected_categorical) > len(deep.selected_categorical)
+    assert set(deep.selected_numeric) <= set(research.selected_numeric)
+
+
+def test_budgeted_deep_statistics_only_passes_triaged_columns(monkeypatch):
+    frame = _wide_frame(rows=4_000)
+    seen: dict[str, object] = {}
+
+    def fake_deep(diagnostic_view, max_pairs=20):
+        seen["columns"] = list(diagnostic_view.columns)
+        seen["rows"] = len(diagnostic_view)
+        seen["pairs"] = max_pairs
+        return {"available": True}
+
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.run_deep_statistics_v2",
+        fake_deep,
+    )
+    budget = derive_execution_budget(len(frame), len(frame.columns), mode="deep")
+    result = run_budgeted_deep_statistics(frame, budget=budget)
+
+    assert len(seen["columns"]) <= 20
+    assert seen["rows"] <= budget.bootstrap_sample_rows
+    assert result["column_triage"]["numeric_limit"] == 12
+    assert result["column_triage"]["categorical_limit"] == 8
+    assert result["execution"]["method"] == "bounded_deep_statistics"
+    assert result["execution"]["scope"] == "bounded_deep_statistics"
+    assert result["execution"]["adaptive_strategy"] == "column_interest_triage"
+    assert result["execution"]["diagnostic_columns"] == len(seen["columns"])
diff --git a/tests/test_fast_anomaly.py b/tests/test_fast_anomaly.py
new file mode 100644
index 0000000..36e3604
--- /dev/null
+++ b/tests/test_fast_anomaly.py
@@ -0,0 +1,37 @@
+import numpy as np
+import pandas as pd
+
+from framevitals.fast_anomaly import fast_anomaly_scan
+
+
+def test_fast_anomaly_scan_detects_injected_multivariate_shift():
+    rng = np.random.default_rng(42)
+    rows = 2_000
+    matrix = rng.normal(size=(rows, 8))
+    matrix[-12:] += 9.0
+    frame = pd.DataFrame(matrix, columns=[f"x{i}" for i in range(8)])
+
+    result = fast_anomaly_scan(frame, contamination=0.02, top_k=25)
+
+    assert result["available"] is True
+    assert result["method"] == "fast_robust_random_projection"
+    assert result["detectors_run"] == [
+        "robust_feature_deviation",
+        "random_projection_tail",
+        "random_projection_density",
+    ]
+    assert result["n_rows_scored"] == rows
+    top_indices = {int(item["row_index"]) for item in result["top_rows"][:20]}
+    assert len(top_indices & set(range(rows - 12, rows))) >= 8
+
+
+def test_fast_anomaly_scan_bounds_numeric_dimensions():
+    rng = np.random.default_rng(7)
+    frame = pd.DataFrame(rng.normal(size=(500, 80)))
+
+    result = fast_anomaly_scan(frame, max_columns=16, projections=8)
+
+    assert result["available"] is True
+    assert len(result["used_columns"]) == 16
+    assert result["preparation"]["truncated_columns"] is True
+    assert result["projection_count"] == 8
diff --git a/tests/test_neural_anomaly.py b/tests/test_neural_anomaly.py
new file mode 100644
index 0000000..53235ba
--- /dev/null
+++ b/tests/test_neural_anomaly.py
@@ -0,0 +1,66 @@
+import numpy as np
+import pandas as pd
+
+from framevitals.budgeted_analysis import run_budgeted_anomalies
+from framevitals.execution import derive_execution_budget
+from framevitals.neural_anomaly import neural_reconstruction_anomalies
+
+
+def _frame(rows: int = 800) -> pd.DataFrame:
+    rng = np.random.default_rng(42)
+    x = rng.normal(size=(rows, 5))
+    x[-5:] += 8.0
+    frame = pd.DataFrame(x, columns=[f"x{i}" for i in range(5)])
+    frame["constant"] = 1.0
+    return frame
+
+
+def test_neural_reconstruction_detector_is_bounded_and_finds_shifted_rows():
+    frame = _frame()
+    result = neural_reconstruction_anomalies(
+        frame,
+        max_rows=500,
+        max_columns=5,
+        max_iter=20,
+        top_k=15,
+    )
+
+    assert result["available"] is True
+    assert result["sample_rows"] <= 500
+    assert result["columns_used"] <= 5
+    assert result["method"] == "bounded_mlp_reconstruction"
+    assert result["architecture"][0] == result["architecture"][-1]
+    assert result["top_rows"]
+
+
+def test_neural_reconstruction_is_research_only(monkeypatch):
+    seen = {"calls": 0}
+
+    def fake_classical(frame, **kwargs):
+        return {"available": True, "n_rows_scored": len(frame)}
+
+    def fake_neural(frame, **kwargs):
+        seen["calls"] += 1
+        return {"available": True, "method": "fake_neural"}
+
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.detect_anomalies_ensemble",
+        fake_classical,
+    )
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.neural_reconstruction_anomalies",
+        fake_neural,
+    )
+
+    frame = _frame(200)
+    deep = derive_execution_budget(len(frame), len(frame.columns), mode="deep")
+    deep_result = run_budgeted_anomalies(frame, budget=deep)
+    assert seen["calls"] == 0
+    assert "neural_reconstruction" not in deep_result
+    assert deep_result["execution"]["neural_reconstruction_enabled"] is False
+
+    research = derive_execution_budget(len(frame), len(frame.columns), mode="research")
+    research_result = run_budgeted_anomalies(frame, budget=research)
+    assert seen["calls"] == 1
+    assert research_result["neural_reconstruction"]["available"] is True
+    assert research_result["execution"]["neural_reconstruction_enabled"] is True
diff --git a/tests/test_stream_change.py b/tests/test_stream_change.py
new file mode 100644
index 0000000..d39c054
--- /dev/null
+++ b/tests/test_stream_change.py
@@ -0,0 +1,96 @@
+import numpy as np
+import pandas as pd
+
+from framevitals.budgeted_analysis import run_budgeted_time_series
+from framevitals.execution import derive_execution_budget
+from framevitals.stream_change import PageHinkleyMeanShift, scan_ordered_mean_shift
+
+
+def test_page_hinkley_detects_sustained_batch_mean_shift():
+    rng = np.random.default_rng(42)
+    detector = PageHinkleyMeanShift(threshold=8.0, min_updates=8)
+
+    for value in rng.normal(0.0, 0.08, size=24):
+        detector.update(float(value))
+    assert detector.detected is False
+
+    for value in rng.normal(2.5, 0.08, size=12):
+        detector.update(float(value))
+        if detector.detected:
+            break
+
+    snapshot = detector.snapshot()
+    assert snapshot["detected"] is True
+    assert snapshot["direction"] == "up"
+    assert snapshot["detected_at_batch"] is not None
+    assert snapshot["sufficient_batches"] is True
+
+
+def test_page_hinkley_stays_quiet_on_stationary_batch_means():
+    rng = np.random.default_rng(7)
+    detector = PageHinkleyMeanShift(threshold=10.0, min_updates=8)
+
+    for value in rng.normal(10.0, 0.15, size=80):
+        detector.update(float(value))
+
+    assert detector.detected is False
+    assert detector.snapshot()["updates"] == 80
+
+
+def test_page_hinkley_ignores_nonfinite_observations():
+    detector = PageHinkleyMeanShift()
+    detector.update(None)
+    detector.update(float("nan"))
+    detector.update(float("inf"))
+
+    assert detector.count == 0
+    assert detector.snapshot()["sufficient_batches"] is False
+
+
+def test_ordered_mean_shift_scan_detects_series_regime_change():
+    rng = np.random.default_rng(123)
+    series = pd.Series(np.concatenate([
+        rng.normal(0.0, 0.1, size=600),
+        rng.normal(3.0, 0.1, size=600),
+    ]))
+
+    result = scan_ordered_mean_shift(series, windows=24)
+
+    assert result["available"] is True
+    assert result["detected"] is True
+    assert result["direction"] == "up"
+    assert result["windows"] == 24
+
+
+def test_budgeted_time_series_attaches_mean_shift_when_series_is_detected(monkeypatch):
+    rows = 1_200
+    rng = np.random.default_rng(5)
+    values = np.concatenate([
+        rng.normal(0.0, 0.1, size=600),
+        rng.normal(2.5, 0.1, size=600),
+    ])
+    frame = pd.DataFrame({
+        "event_time": pd.date_range("2025-01-01", periods=rows, freq="h"),
+        "value": values,
+    })
+
+    def fake_time_series(work, target_column=None):
+        return {
+            "available": True,
+            "detected_date_column": "event_time",
+            "numeric_column": "value",
+        }
+
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.detect_and_analyze_time_series",
+        fake_time_series,
+    )
+    budget = derive_execution_budget(rows, 2, mode="standard")
+    result = run_budgeted_time_series(frame, budget=budget)
+
+    assert result["mean_shift"]["available"] is True
+    assert result["mean_shift"]["detected"] is True
+    assert result["execution"]["mean_shift_detection_enabled"] is True
+    assert result["execution"]["method"] == "bounded_time_series"
+    assert result["execution"]["scope"] == "bounded_time_series"
+    assert result["execution"]["adaptive_strategy"] == "ordered_page_hinkley_mean_shift"
diff --git a/tests/test_streaming_sketches.py b/tests/test_streaming_sketches.py
new file mode 100644
index 0000000..320a461
--- /dev/null
+++ b/tests/test_streaming_sketches.py
@@ -0,0 +1,44 @@
+import numpy as np
+import pytest
+
+from framevitals.streaming_sketches import (
+    NumpyLogQuantileSketch,
+    PYTHON_NUMERIC_SKETCH_CELL_BUDGET,
+    should_use_full_stream_numpy_sketch,
+)
+
+
+def test_numpy_log_quantile_sketch_is_mergeable_and_bounded():
+    values = np.arange(1, 10_001, dtype=np.float64)
+    left = NumpyLogQuantileSketch().update(values[:5_000])
+    right = NumpyLogQuantileSketch().update(values[5_000:])
+    merged = left.merge(right)
+
+    assert merged.count == 10_000
+    assert merged.quantile(0.5) == pytest.approx(5_000, rel=0.03)
+    assert merged.quantile(0.95) == pytest.approx(9_500, rel=0.03)
+    assert merged.bin_count < 1_000
+    snapshot = merged.snapshot()
+    assert snapshot["method"] == "numpy_log_quantile_sketch"
+    assert snapshot["relative_accuracy"] == 0.01
+
+
+def test_numpy_log_quantile_sketch_handles_signed_zero_and_nonfinite_values():
+    sketch = NumpyLogQuantileSketch().update(
+        np.array([-100.0, -10.0, -0.0, 0.0, 10.0, 100.0, np.nan, np.inf])
+    )
+
+    assert sketch.count == 6
+    assert sketch.zero_count == 2
+    assert sketch.quantile(0.0) < 0
+    assert sketch.quantile(1.0) > 0
+
+
+def test_numpy_stream_sketch_budget_uses_full_stream_for_normal_numeric_workloads():
+    assert should_use_full_stream_numpy_sketch(100_000, 100) is True
+    assert 100_000 * 100 <= PYTHON_NUMERIC_SKETCH_CELL_BUDGET
+
+
+def test_numpy_stream_sketch_budget_avoids_ultra_wide_cpu_regression():
+    assert should_use_full_stream_numpy_sketch(100_000, 6_750) is False
+    assert should_use_full_stream_numpy_sketch(100_000, 10_000) is False

From 3c1f6306df885b5f8ba32e971cce1d7219d57ea9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:12:13 +0530
Subject: [PATCH 380/439] perf: replace deep bootstrap hot path with fast
 confidence intervals

---
 src/framevitals/budgeted_analysis.py    |  15 +-
 src/framevitals/fast_deep_statistics.py | 269 ++++++++++++++++++++++++
 tests/test_adaptive_execution.py        |  28 ++-
 tests/test_fast_deep_statistics.py      |  58 +++++
 4 files changed, 367 insertions(+), 3 deletions(-)
 create mode 100644 src/framevitals/fast_deep_statistics.py
 create mode 100644 tests/test_fast_deep_statistics.py

diff --git a/src/framevitals/budgeted_analysis.py b/src/framevitals/budgeted_analysis.py
index b8f50c9..8ed850f 100644
--- a/src/framevitals/budgeted_analysis.py
+++ b/src/framevitals/budgeted_analysis.py
@@ -13,6 +13,7 @@
 
 from framevitals.anomaly_ensemble import detect_anomalies_ensemble
 from framevitals.deep_statistics_v2 import run_deep_statistics_v2
+from framevitals.fast_deep_statistics import run_fast_deep_statistics_v2
 from framevitals.deep_triage import triage_deep_columns
 from framevitals.execution import ExecutionBudget, deterministic_sample_frame
 from framevitals.fast_anomaly import fast_anomaly_scan
@@ -75,7 +76,15 @@ def run_budgeted_deep_statistics(
     if pair_budget < 1:
         raise ValueError("max_pairs must be at least 1.")
 
-    payload = run_deep_statistics_v2(diagnostic_view, max_pairs=pair_budget)
+    if budget.mode == "research":
+        payload = run_deep_statistics_v2(diagnostic_view, max_pairs=pair_budget)
+        inference_strategy = "bca_bootstrap"
+    else:
+        payload = run_fast_deep_statistics_v2(
+            diagnostic_view,
+            max_pairs=pair_budget,
+        )
+        inference_strategy = "closed_form_and_order_statistics"
     triage_payload = triage.to_dict()
     payload["column_triage"] = triage_payload
 
@@ -83,13 +92,15 @@ def run_budgeted_deep_statistics(
         **sampling,
         "reason": (
             "Deep statistics use a bounded row view plus adaptive column triage; "
-            "bootstrap/distribution work is reserved for the highest-interest columns."
+            "expensive distribution work is reserved for the highest-interest columns; "
+            "non-research modes use O(n) confidence intervals instead of resampling."
         ),
         "pair_budget": int(pair_budget),
         "column_triage": triage_payload,
         "source_columns": int(dataframe.shape[1]),
         "diagnostic_columns": int(len(selected_columns)),
         "adaptive_strategy": "column_interest_triage",
+        "inference_strategy": inference_strategy,
     }
     return _attach_execution(
         payload,
diff --git a/src/framevitals/fast_deep_statistics.py b/src/framevitals/fast_deep_statistics.py
new file mode 100644
index 0000000..eccd0d3
--- /dev/null
+++ b/src/framevitals/fast_deep_statistics.py
@@ -0,0 +1,269 @@
+"""Fast deep-statistics execution for non-research analysis modes.
+
+The public ``deep_statistics_v2`` implementation keeps its BCa bootstrap
+semantics.  Budgeted quick/standard/deep execution can use this module to avoid
+thousands of resamples per numeric column while preserving the rest of the v2
+diagnostic battery.
+
+Mean confidence intervals use the exact Student-t construction and median
+confidence intervals use distribution-free order statistics derived from the
+Binomial(n, 0.5) model.  Both are deterministic and O(n).
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+from framevitals.deep_statistics_v2 import (
+    _categorical_column_stats,
+    _classify_kurtosis,
+    _classify_skew,
+    _fit_best_distribution,
+    _group_difference_test,
+    _normality,
+    _numeric_pair_stats,
+    _outlier_flags,
+    _point_biserial,
+    _safe_float,
+    run_deep_statistics_v2,
+)
+
+
+def _finite_numeric(series: pd.Series) -> np.ndarray:
+    values = pd.to_numeric(series, errors="coerce").to_numpy(
+        dtype="float64",
+        na_value=np.nan,
+    )
+    return values[np.isfinite(values)]
+
+
+def fast_mean_ci(
+    series: pd.Series,
+    confidence: float = 0.95,
+) -> dict[str, Any]:
+    """Student-t confidence interval for the arithmetic mean in O(n)."""
+    values = _finite_numeric(series)
+    n = int(values.size)
+    if n < 20:
+        return {"available": False, "reason": "n<20"}
+
+    mean = float(values.mean())
+    std = float(values.std(ddof=1))
+    if not math.isfinite(std):
+        return {"available": False, "reason": "non-finite sample variance"}
+    if std <= 1.0e-15:
+        return {
+            "available": True,
+            "low": _safe_float(mean),
+            "high": _safe_float(mean),
+            "method": "student_t",
+            "n_resamples": 0,
+        }
+
+    alpha = 1.0 - float(confidence)
+    critical = float(stats.t.ppf(1.0 - alpha / 2.0, n - 1))
+    half_width = critical * std / math.sqrt(n)
+    return {
+        "available": True,
+        "low": _safe_float(mean - half_width),
+        "high": _safe_float(mean + half_width),
+        "method": "student_t",
+        "n_resamples": 0,
+    }
+
+
+def fast_median_ci(
+    series: pd.Series,
+    confidence: float = 0.95,
+) -> dict[str, Any]:
+    """Distribution-free median CI via two selected order statistics.
+
+    ``np.partition`` selects only the two required ranks, so no full sort or
+    resampling matrix is allocated.
+    """
+    values = _finite_numeric(series)
+    n = int(values.size)
+    if n < 20:
+        return {"available": False, "reason": "n<20"}
+
+    alpha = 1.0 - float(confidence)
+    rank = int(stats.binom.ppf(alpha / 2.0, n, 0.5))
+    rank = max(1, min(rank, n // 2))
+
+    low_index = rank - 1
+    high_index = n - rank
+    partitioned = np.partition(values, (low_index, high_index))
+
+    return {
+        "available": True,
+        "low": _safe_float(partitioned[low_index]),
+        "high": _safe_float(partitioned[high_index]),
+        "method": "distribution_free_order_statistic",
+        "n_resamples": 0,
+        "rank_low": int(rank),
+        "rank_high": int(n - rank + 1),
+    }
+
+
+def _numeric_column_stats_fast(series: pd.Series) -> dict[str, Any]:
+    s = series.dropna()
+    if s.empty:
+        return {"status": "empty"}
+
+    skew = _safe_float(s.skew())
+    kurt = _safe_float(s.kurtosis())
+    q1, q3 = _safe_float(s.quantile(0.25)), _safe_float(s.quantile(0.75))
+    iqr = (q3 - q1) if (q1 is not None and q3 is not None) else None
+
+    return {
+        "count": int(s.count()),
+        "mean": _safe_float(s.mean()),
+        "median": _safe_float(s.median()),
+        "std": _safe_float(s.std()),
+        "min": _safe_float(s.min()),
+        "max": _safe_float(s.max()),
+        "q1": q1,
+        "q3": q3,
+        "iqr": _safe_float(iqr),
+        "skewness": skew,
+        "skewness_label": _classify_skew(skew),
+        "kurtosis": kurt,
+        "kurtosis_label": _classify_kurtosis(kurt),
+        "outliers": _outlier_flags(series),
+        "normality": _normality(series),
+        "distribution_fit": _fit_best_distribution(series),
+        "bootstrap_mean_ci": fast_mean_ci(series),
+        "bootstrap_median_ci": fast_median_ci(series),
+    }
+
+
+def _numeric_bivariate(
+    df: pd.DataFrame,
+    numeric_cols: list[str],
+    categorical_cols: list[str],
+    *,
+    max_pairs: int,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
+    numeric_pairs: list[dict[str, Any]] = []
+    budget = max_pairs
+    for index, left in enumerate(numeric_cols):
+        for right in numeric_cols[index + 1 :]:
+            if budget <= 0:
+                break
+            payload = _numeric_pair_stats(df[left], df[right])
+            if payload.get("available"):
+                numeric_pairs.append(
+                    {"column_a": left, "column_b": right, **payload}
+                )
+                budget -= 1
+        if budget <= 0:
+            break
+    numeric_pairs.sort(
+        key=lambda item: abs(item.get("pearson", {}).get("r") or 0),
+        reverse=True,
+    )
+
+    binary_numeric_pairs: list[dict[str, Any]] = []
+    budget = max_pairs
+    for category in categorical_cols:
+        if df[category].nunique(dropna=True) != 2:
+            continue
+        for numeric in numeric_cols:
+            if budget <= 0:
+                break
+            payload = _point_biserial(df[category], df[numeric])
+            if payload.get("available"):
+                binary_numeric_pairs.append(
+                    {
+                        "binary_column": category,
+                        "numeric_column": numeric,
+                        **payload,
+                    }
+                )
+                budget -= 1
+        if budget <= 0:
+            break
+
+    group_difference_tests: list[dict[str, Any]] = []
+    budget = max_pairs
+    for category in categorical_cols:
+        unique = df[category].nunique(dropna=True)
+        if unique < 2 or unique > 6:
+            continue
+        for numeric in numeric_cols:
+            if budget <= 0:
+                break
+            payload = _group_difference_test(df[numeric], df[category])
+            if payload.get("available"):
+                group_difference_tests.append(
+                    {
+                        "group_column": category,
+                        "numeric_column": numeric,
+                        **payload,
+                    }
+                )
+                budget -= 1
+        if budget <= 0:
+            break
+
+    return numeric_pairs, binary_numeric_pairs, group_difference_tests
+
+
+def run_fast_deep_statistics_v2(
+    df: pd.DataFrame,
+    max_pairs: int = 20,
+) -> dict[str, Any]:
+    """Run the v2 diagnostic battery with O(n) confidence intervals."""
+    numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    categorical_cols = df.select_dtypes(
+        include=["object", "category", "bool"]
+    ).columns.tolist()
+
+    categorical_view = (
+        df.loc[:, categorical_cols]
+        if categorical_cols
+        else pd.DataFrame(index=df.index)
+    )
+    result = run_deep_statistics_v2(categorical_view, max_pairs=max_pairs)
+
+    result["numeric_columns"] = numeric_cols
+    result["numeric_statistics"] = {
+        column: _numeric_column_stats_fast(df[column])
+        for column in numeric_cols
+    }
+
+    (
+        numeric_pairs,
+        binary_numeric_pairs,
+        group_difference_tests,
+    ) = _numeric_bivariate(
+        df,
+        numeric_cols,
+        categorical_cols,
+        max_pairs=max_pairs,
+    )
+
+    result["bivariate"]["numeric_pairs"] = numeric_pairs
+    result["bivariate"]["binary_numeric_pairs"] = binary_numeric_pairs
+    result["bivariate"]["group_difference_tests"] = group_difference_tests
+
+    result["summary"].update(
+        {
+            "numeric_count": len(numeric_cols),
+            "numeric_pairs_tested": len(numeric_pairs),
+            "binary_numeric_pairs_tested": len(binary_numeric_pairs),
+            "group_difference_tests_run": len(group_difference_tests),
+        }
+    )
+    result["inference"] = {
+        "method": "fast_closed_form_and_order_statistics",
+        "mean_ci": "student_t",
+        "median_ci": "distribution_free_order_statistic",
+        "bootstrap_resamples": 0,
+    }
+    return result
diff --git a/tests/test_adaptive_execution.py b/tests/test_adaptive_execution.py
index d3bad1d..97346f2 100644
--- a/tests/test_adaptive_execution.py
+++ b/tests/test_adaptive_execution.py
@@ -84,7 +84,7 @@ def fake_deep(frame, max_pairs=20):
         return {"available": True}
 
     monkeypatch.setattr(
-        "framevitals.budgeted_analysis.run_deep_statistics_v2",
+        "framevitals.budgeted_analysis.run_fast_deep_statistics_v2",
         fake_deep,
     )
     frame = pd.DataFrame({"x": np.arange(20_000), "y": np.arange(20_000)})
@@ -96,6 +96,32 @@ def fake_deep(frame, max_pairs=20):
     assert seen["pairs"] <= budget.relationship_pair_budget
     assert result["execution"]["sampled"] is True
     assert result["execution"]["source_rows"] == 20_000
+    assert (
+        result["execution"]["inference_strategy"]
+        == "closed_form_and_order_statistics"
+    )
+
+
+def test_research_deep_statistics_adapter_keeps_bca(monkeypatch):
+    seen = {}
+
+    def fake_deep(frame, max_pairs=20):
+        seen["rows"] = len(frame)
+        seen["pairs"] = max_pairs
+        return {"available": True}
+
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.run_deep_statistics_v2",
+        fake_deep,
+    )
+    frame = pd.DataFrame({"x": np.arange(500), "y": np.arange(500)})
+    budget = derive_execution_budget(len(frame), len(frame.columns), mode="research")
+
+    result = run_budgeted_deep_statistics(frame, budget=budget)
+
+    assert seen["rows"] == len(frame)
+    assert seen["pairs"] <= budget.relationship_pair_budget
+    assert result["execution"]["inference_strategy"] == "bca_bootstrap"
 
 
 def test_anomaly_adapter_discloses_sample_coverage(monkeypatch):
diff --git a/tests/test_fast_deep_statistics.py b/tests/test_fast_deep_statistics.py
new file mode 100644
index 0000000..c4d8553
--- /dev/null
+++ b/tests/test_fast_deep_statistics.py
@@ -0,0 +1,58 @@
+import numpy as np
+import pandas as pd
+
+from framevitals.fast_deep_statistics import (
+    fast_mean_ci,
+    fast_median_ci,
+    run_fast_deep_statistics_v2,
+)
+
+
+def test_fast_mean_ci_matches_normal_sample_location():
+    rng = np.random.default_rng(42)
+    series = pd.Series(rng.normal(loc=3.0, scale=1.5, size=2_000))
+
+    result = fast_mean_ci(series)
+
+    assert result["available"] is True
+    assert result["method"] == "student_t"
+    assert result["n_resamples"] == 0
+    assert result["low"] < series.mean() < result["high"]
+
+
+def test_fast_median_ci_is_distribution_free_and_contains_median():
+    rng = np.random.default_rng(7)
+    series = pd.Series(rng.lognormal(mean=0.3, sigma=1.1, size=2_000))
+
+    result = fast_median_ci(series)
+
+    assert result["available"] is True
+    assert result["method"] == "distribution_free_order_statistic"
+    assert result["n_resamples"] == 0
+    assert result["rank_low"] < result["rank_high"]
+    assert result["low"] <= series.median() <= result["high"]
+
+
+def test_fast_deep_statistics_preserves_v2_shape():
+    rng = np.random.default_rng(11)
+    frame = pd.DataFrame({
+        "x": rng.normal(size=120),
+        "y": rng.gamma(shape=2.0, scale=1.0, size=120),
+        "group": np.where(np.arange(120) % 2, "A", "B"),
+    })
+
+    result = run_fast_deep_statistics_v2(frame, max_pairs=5)
+
+    assert result["version"] == "v2"
+    assert result["numeric_columns"] == ["x", "y"]
+    assert result["categorical_columns"] == ["group"]
+    assert result["numeric_statistics"]["x"]["bootstrap_mean_ci"]["n_resamples"] == 0
+    assert result["numeric_statistics"]["x"]["bootstrap_median_ci"]["n_resamples"] == 0
+    assert result["inference"]["bootstrap_resamples"] == 0
+
+
+def test_fast_ci_rejects_tiny_samples_like_existing_bootstrap_contract():
+    series = pd.Series(np.arange(10, dtype=float))
+
+    assert fast_mean_ci(series) == {"available": False, "reason": "n<20"}
+    assert fast_median_ci(series) == {"available": False, "reason": "n<20"}

From 6b5e1be3680ffe5f4becb8069b5bddf66b2578b9 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:14:47 +0530
Subject: [PATCH 381/439] test: update deep triage hook for fast inference

---
 tests/test_deep_triage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_deep_triage.py b/tests/test_deep_triage.py
index d366e06..83b9642 100644
--- a/tests/test_deep_triage.py
+++ b/tests/test_deep_triage.py
@@ -54,7 +54,7 @@ def fake_deep(diagnostic_view, max_pairs=20):
         return {"available": True}
 
     monkeypatch.setattr(
-        "framevitals.budgeted_analysis.run_deep_statistics_v2",
+        "framevitals.budgeted_analysis.run_fast_deep_statistics_v2",
         fake_deep,
     )
     budget = derive_execution_budget(len(frame), len(frame.columns), mode="deep")

From 1d772284922c247e3a56fe3fafc15aaf6303d4ad Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:16:44 +0530
Subject: [PATCH 382/439] bench: add reproducible 8k x 180 deep performance
 target

---
 .github/workflows/benchmark.yml       |  67 +++++++++++++-
 benchmarks/benchmark_deep_pipeline.py | 122 ++++++++++++++++++++++++++
 2 files changed, 187 insertions(+), 2 deletions(-)
 create mode 100644 benchmarks/benchmark_deep_pipeline.py

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index e6fba7a..481b32c 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -2,11 +2,15 @@ name: Performance Benchmark
 
 on:
   push:
-    branches: [main]
+    branches: [main, "develop/august"]
+    paths:
+      - "src/framevitals/**"
+      - "benchmarks/**"
+      - ".github/workflows/benchmark.yml"
   workflow_dispatch:
     inputs:
       rows:
-        description: Rows in the synthetic benchmark workload
+        description: Rows in the synthetic profile benchmark workload
         required: false
         default: "50000"
 
@@ -87,3 +91,62 @@ jobs:
           name: framevitals-profile-benchmark
           path: benchmark-results.json
           retention-days: 30
+
+  deep-8k-180:
+    name: 8k x 180 Deep benchmark
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install benchmark runtime
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e "."
+
+      - name: Run reproducible 8k x 180 Deep benchmark
+        env:
+          FRAMEVITALS_BACKEND: numpy
+        run: |
+          python benchmarks/benchmark_deep_pipeline.py \
+            --rows 8000 \
+            --columns 180 \
+            --output deep-benchmark-results.json
+
+      - name: Publish Deep benchmark summary
+        run: |
+          python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
+          import json
+          from pathlib import Path
+
+          payload = json.loads(Path("deep-benchmark-results.json").read_text())
+          print("## FrameVitals 8k x 180 Deep benchmark")
+          print()
+          print(f"- Wall time: **{payload['elapsed_seconds']:.3f}s**")
+          print(f"- Historical original: {payload['historical_original_seconds']:.3f}s")
+          print(f"- Speedup vs historical original: **{payload['speedup_vs_historical_original']:.2f}x**")
+          print(f"- 10x target: {payload['ten_x_target_seconds']:.3f}s")
+          print(f"- Target met: **{payload['ten_x_target_met']}**")
+          print()
+          print("### Pipeline stages")
+          for name, milliseconds in sorted(payload["pipeline_timings_ms"].items()):
+              if isinstance(milliseconds, (int, float)):
+                  print(f"- {name}: {milliseconds / 1000:.3f}s")
+          PY
+
+      - name: Upload Deep benchmark result
+        uses: actions/upload-artifact@v6
+        with:
+          name: framevitals-deep-8k-180-benchmark
+          path: deep-benchmark-results.json
+          retention-days: 30
diff --git a/benchmarks/benchmark_deep_pipeline.py b/benchmarks/benchmark_deep_pipeline.py
new file mode 100644
index 0000000..4026f04
--- /dev/null
+++ b/benchmarks/benchmark_deep_pipeline.py
@@ -0,0 +1,122 @@
+"""Reproducible 8k x 180 Deep pipeline benchmark.
+
+This workload is the stable performance target for the August optimization
+track.  It is intentionally numeric and wide so profiling, adaptive deep
+statistics, anomaly screening, and pairwise diagnostics all receive work.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import time
+
+import numpy as np
+import pandas as pd
+
+from framevitals.pipeline import run_full_analysis
+
+
+DEFAULT_ROWS = 8_000
+DEFAULT_COLUMNS = 180
+HISTORICAL_ORIGINAL_SECONDS = 76.17
+TEN_X_TARGET_SECONDS = HISTORICAL_ORIGINAL_SECONDS / 10.0
+
+
+def make_workload(
+    rows: int = DEFAULT_ROWS,
+    columns: int = DEFAULT_COLUMNS,
+    *,
+    seed: int = 42,
+) -> pd.DataFrame:
+    if rows < 100 or columns < 32:
+        raise ValueError("benchmark requires at least 100 rows and 32 columns")
+
+    rng = np.random.default_rng(seed)
+    values = rng.normal(size=(rows, columns))
+
+    # Make a bounded set of columns diagnostically interesting so deep triage
+    # consistently exercises skew, missingness, constants, and relationships.
+    skewed = min(12, columns)
+    values[:, :skewed] = rng.lognormal(
+        mean=0.0,
+        sigma=1.5,
+        size=(rows, skewed),
+    )
+
+    for index in range(12, min(24, columns)):
+        values[::11, index] = np.nan
+
+    for index in range(24, min(28, columns)):
+        values[:, index] = float(index)
+
+    for offset, index in enumerate(range(30, min(40, columns))):
+        source = offset % min(10, columns)
+        values[:, index] = values[:, source] * 0.97 + rng.normal(
+            scale=0.03,
+            size=rows,
+        )
+
+    return pd.DataFrame(
+        values,
+        columns=[f"n{index:03d}" for index in range(columns)],
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--rows", type=int, default=DEFAULT_ROWS)
+    parser.add_argument("--columns", type=int, default=DEFAULT_COLUMNS)
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+
+    os.environ.setdefault("FRAMEVITALS_BACKEND", "numpy")
+    frame = make_workload(args.rows, args.columns)
+
+    started = time.perf_counter()
+    result = run_full_analysis(
+        "benchmark-8k-180-deep",
+        original_filename="synthetic-8k-180",
+        analysis_mode="deep",
+        skip_ai=True,
+        parallel_workers=4,
+        dataframe=frame,
+        write_artifacts=False,
+    )
+    elapsed = time.perf_counter() - started
+
+    payload = {
+        "benchmark_schema_version": 1,
+        "workload": {
+            "rows": int(args.rows),
+            "columns": int(args.columns),
+            "kind": "deterministic_wide_numeric_deep",
+            "seed": 42,
+            "backend": os.environ.get("FRAMEVITALS_BACKEND", "auto"),
+        },
+        "elapsed_seconds": round(float(elapsed), 6),
+        "historical_original_seconds": HISTORICAL_ORIGINAL_SECONDS,
+        "ten_x_target_seconds": round(TEN_X_TARGET_SECONDS, 6),
+        "speedup_vs_historical_original": round(
+            HISTORICAL_ORIGINAL_SECONDS / max(elapsed, 1.0e-12),
+            4,
+        ),
+        "ten_x_target_met": bool(elapsed <= TEN_X_TARGET_SECONDS),
+        "pipeline_timings_ms": result.get("timings_ms", {}),
+        "deep_execution": (
+            result.get("deep_statistics_v2", {}).get("execution", {})
+            if isinstance(result.get("deep_statistics_v2"), dict)
+            else {}
+        ),
+    }
+
+    encoded = json.dumps(payload, indent=2, sort_keys=True)
+    print(encoded)
+    if args.output is not None:
+        args.output.write_text(encoded + "\n", encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main()

From b28b25320b82e0f836c6d93af10191e94315adaa Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:23:54 +0530
Subject: [PATCH 383/439] perf: bound ultra-wide streaming analysis by column
 budget

---
 .../workflows/extreme-streaming-benchmark.yml |  88 +++++++++
 benchmarks/benchmark_extreme_streaming.py     | 154 +++++++++++++++
 src/framevitals/execution.py                  |  55 ++++++
 src/framevitals/health_score.py               |  54 ++++--
 src/framevitals/planning_api.py               | 100 +++++++++-
 src/framevitals/streaming_pipeline.py         | 178 +++++++++++++++++-
 tests/test_extreme_streaming_projection.py    | 107 +++++++++++
 7 files changed, 709 insertions(+), 27 deletions(-)
 create mode 100644 .github/workflows/extreme-streaming-benchmark.yml
 create mode 100644 benchmarks/benchmark_extreme_streaming.py
 create mode 100644 tests/test_extreme_streaming_projection.py

diff --git a/.github/workflows/extreme-streaming-benchmark.yml b/.github/workflows/extreme-streaming-benchmark.yml
new file mode 100644
index 0000000..92434fc
--- /dev/null
+++ b/.github/workflows/extreme-streaming-benchmark.yml
@@ -0,0 +1,88 @@
+name: Extreme Streaming Benchmark
+
+on:
+  push:
+    branches: ["develop/august"]
+    paths:
+      - "src/framevitals/execution.py"
+      - "src/framevitals/planning_api.py"
+      - "src/framevitals/streaming_pipeline.py"
+      - "src/framevitals/health_score.py"
+      - "benchmarks/benchmark_extreme_streaming.py"
+      - ".github/workflows/extreme-streaming-benchmark.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  deep-1m-x-100k:
+    name: Deep 1M x 100k virtual stream
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install streaming benchmark runtime
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[arrow]"
+
+      - name: Run 1M x 100k Deep benchmark
+        env:
+          FRAMEVITALS_BACKEND: numpy
+        run: python benchmarks/benchmark_extreme_streaming.py
+
+      - name: Publish benchmark summary
+        if: always()
+        run: |
+          if [[ ! -f extreme-benchmark-results.json ]]; then
+            exit 0
+          fi
+          python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
+          import json
+          from pathlib import Path
+
+          payload = json.loads(Path("extreme-benchmark-results.json").read_text())
+          workload = payload["workload"]
+          safety = payload["safety"]
+          print("## FrameVitals 1M x 100k Deep benchmark")
+          print()
+          print(f"- Logical cells: **{workload['cells']:,}**")
+          print(f"- Dense float64 equivalent: **{workload['dense_float64_raw_gb']:.1f} GB raw**")
+          print(f"- Plan: **{payload['plan_seconds']:.3f}s**")
+          print(f"- Deep analysis: **{payload['analysis_seconds']:.3f}s**")
+          print(f"- Scale class: **{payload['scale_class']}**")
+          print(f"- Profiled columns: **{safety['profiled_columns']} / {safety['source_columns']}**")
+          print(f"- All 1,000,000 source rows scanned: **{safety['all_source_rows_scanned']}**")
+          print(f"- Full materialization: **{safety['full_materialization']}**")
+          print(f"- Unbounded 100k-column request: **{safety['unbounded_width_requested']}**")
+          print()
+          print("### Pipeline stages")
+          for name, milliseconds in sorted(payload.get("pipeline_timings_ms", {}).items()):
+              if isinstance(milliseconds, (int, float)):
+                  print(f"- {name}: {milliseconds / 1000:.3f}s")
+          PY
+
+      - name: Upload benchmark result
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: framevitals-extreme-1m-x-100k
+          path: extreme-benchmark-results.json
+          if-no-files-found: ignore
+          retention-days: 30
diff --git a/benchmarks/benchmark_extreme_streaming.py b/benchmarks/benchmark_extreme_streaming.py
new file mode 100644
index 0000000..24d9ab1
--- /dev/null
+++ b/benchmarks/benchmark_extreme_streaming.py
@@ -0,0 +1,154 @@
+"""Virtual 1,000,000 x 100,000 Deep streaming benchmark.
+
+A dense float64 materialization of this shape would require roughly 800 GB just
+for raw cells, so the benchmark models a projection-capable streaming source.
+FrameVitals must preserve the true source shape while never requesting the full
+100,000-column width from the source.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import time
+from types import SimpleNamespace
+
+import numpy as np
+import pyarrow as pa
+
+import framevitals as fv
+from framevitals.sources import DatasetMetadata
+
+
+ROWS = 1_000_000
+COLUMNS = 100_000
+
+
+class VirtualSchema:
+    def __init__(self, columns: int):
+        self.columns = int(columns)
+        self.dtype = pa.float64()
+
+    def __iter__(self):
+        for index in range(self.columns):
+            yield SimpleNamespace(name=f"n{index:05d}", type=self.dtype)
+
+    def field(self, name: str):
+        index = int(name[1:])
+        if index < 0 or index >= self.columns:
+            raise KeyError(name)
+        return SimpleNamespace(name=name, type=self.dtype)
+
+
+class VirtualExtremeSource:
+    def __init__(self, rows: int = ROWS, columns: int = COLUMNS):
+        self.rows = int(rows)
+        self.columns = int(columns)
+        self.schema_view = VirtualSchema(columns)
+        self.max_requested_columns = 0
+        self.unbounded_requests = 0
+        self.batches_yielded = 0
+        self.rows_yielded = 0
+
+    def inspect(self):
+        return DatasetMetadata(
+            name="virtual-1m-x-100k",
+            kind="virtual",
+            format="synthetic",
+            rows=self.rows,
+            columns=self.columns,
+            size_bytes=self.rows * self.columns * 8,
+            materialized=False,
+            supports_projection=True,
+            supports_streaming=True,
+        )
+
+    def schema(self):
+        return self.schema_view
+
+    def iter_batches(self, *, batch_size=65_536, columns=None):
+        if columns is None:
+            self.unbounded_requests += 1
+            raise RuntimeError(
+                "FrameVitals attempted an unbounded 100,000-column scan."
+            )
+
+        names = list(columns)
+        self.max_requested_columns = max(self.max_requested_columns, len(names))
+        offset = 0
+        while offset < self.rows:
+            take = min(int(batch_size), self.rows - offset)
+            base = np.arange(offset, offset + take, dtype=np.float64)
+            values = pa.array((base % 10_007) / 10_007.0)
+            batch = pa.RecordBatch.from_arrays([values] * len(names), names=names)
+            self.batches_yielded += 1
+            self.rows_yielded += take
+            yield batch
+            offset += take
+
+    def load(self):
+        raise RuntimeError("Virtual extreme source must never materialize fully.")
+
+
+def main() -> None:
+    source = VirtualExtremeSource()
+
+    plan_started = time.perf_counter()
+    plan = fv.plan(source, mode="deep", workers=4)
+    plan_seconds = time.perf_counter() - plan_started
+
+    # Reset counters so execution accounting is independent of planning.
+    source.max_requested_columns = 0
+    source.unbounded_requests = 0
+    source.batches_yielded = 0
+    source.rows_yielded = 0
+
+    analysis_started = time.perf_counter()
+    result = fv.analyze(
+        source,
+        mode="deep",
+        artifacts=False,
+        workers=4,
+    )
+    analysis_seconds = time.perf_counter() - analysis_started
+
+    streaming = result.get("execution", {}).get("streaming", {})
+    payload = {
+        "benchmark_schema_version": 1,
+        "workload": {
+            "rows": ROWS,
+            "columns": COLUMNS,
+            "cells": ROWS * COLUMNS,
+            "dense_float64_raw_gb": round(ROWS * COLUMNS * 8 / 1_000_000_000, 3),
+            "mode": "deep",
+            "source": "virtual_projection_capable_stream",
+        },
+        "plan_seconds": round(plan_seconds, 6),
+        "analysis_seconds": round(analysis_seconds, 6),
+        "scale_class": plan.get("execution_budget", {}).get("scale_class"),
+        "planning": plan.get("planning_data", {}),
+        "streaming_execution": streaming,
+        "pipeline_timings_ms": result.get("timings_ms", {}),
+        "source_observed": {
+            "max_requested_columns": source.max_requested_columns,
+            "unbounded_requests": source.unbounded_requests,
+            "batches_yielded": source.batches_yielded,
+            "rows_yielded": source.rows_yielded,
+        },
+        "safety": {
+            "full_materialization": streaming.get("full_materialization"),
+            "column_sampled": streaming.get("column_sampled"),
+            "profiled_columns": streaming.get("profiled_columns"),
+            "source_columns": streaming.get("source_columns"),
+            "all_source_rows_scanned": source.rows_yielded == ROWS,
+            "unbounded_width_requested": source.unbounded_requests > 0,
+        },
+    }
+
+    output = Path("extreme-benchmark-results.json")
+    output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(payload, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/src/framevitals/execution.py b/src/framevitals/execution.py
index fafb215..836de9b 100644
--- a/src/framevitals/execution.py
+++ b/src/framevitals/execution.py
@@ -20,6 +20,23 @@
 
 _VALID_MODES = {"quick", "standard", "deep", "research"}
 
+# Full-stream profiling is valuable, but on ultra-wide sources scanning every
+# cell defeats the purpose of streaming. These budgets cap the number of source
+# cells inspected by the reusable profile pass while preserving the true source
+# shape in execution metadata.
+_STREAMING_PROFILE_CELL_BUDGETS = {
+    "quick": 64_000_000,
+    "standard": 96_000_000,
+    "deep": 128_000_000,
+    "research": 256_000_000,
+}
+_STREAMING_PROFILE_COLUMN_CAPS = {
+    "quick": 64,
+    "standard": 96,
+    "deep": 128,
+    "research": 256,
+}
+
 
 @dataclass(frozen=True, slots=True)
 class ExecutionBudget:
@@ -58,6 +75,44 @@ def _bounded(requested: int, rows: int) -> int:
     return min(int(requested), int(rows))
 
 
+def derive_streaming_profile_column_limit(
+    rows: int,
+    columns: int,
+    *,
+    mode: str = "standard",
+) -> int:
+    """Return the deterministic full-stream column budget for a source shape.
+
+    Ordinary datasets keep every column. Ultra-wide/high-cell-count sources are
+    projected before the full streaming profile pass so total scanned cells stay
+    bounded. The projection itself is selected by the caller from the source
+    schema; this function only resolves the allowed width.
+    """
+    if mode not in _VALID_MODES:
+        raise ValueError(f"Unknown analysis mode: {mode}")
+    if rows < 0 or columns < 0:
+        raise ValueError("rows and columns must be non-negative.")
+    if columns == 0:
+        return 0
+    if rows == 0:
+        return int(columns)
+
+    cells = int(rows) * int(columns)
+    cell_budget = int(_STREAMING_PROFILE_CELL_BUDGETS[mode])
+    if cells <= cell_budget and columns < 10_000:
+        return int(columns)
+
+    by_cells = max(1, cell_budget // max(int(rows), 1))
+    return max(
+        1,
+        min(
+            int(columns),
+            int(by_cells),
+            int(_STREAMING_PROFILE_COLUMN_CAPS[mode]),
+        ),
+    )
+
+
 def derive_execution_budget(
     rows: int,
     columns: int,
diff --git a/src/framevitals/health_score.py b/src/framevitals/health_score.py
index 0e0f87e..71ec093 100644
--- a/src/framevitals/health_score.py
+++ b/src/framevitals/health_score.py
@@ -143,20 +143,28 @@ def calculate_health_score_from_profile_sample(
 ) -> dict[str, Any]:
     """Calculate a bounded health score from a full profile plus row sample.
 
-    This is intended for streaming sources. Completeness and duplicate metrics
-    come from the full streaming profile, while outlier safety is estimated from
-    the bounded sample. Constant/high-cardinality signals use full-stream profile
-    summaries when available.
+    On ultra-wide sources the profile may cover all rows but a deterministic
+    subset of columns. Completeness and consistency are then estimates over that
+    projected column sample rather than being diluted by the unprofiled width.
     """
     rows = max(int(profile.get("shape", {}).get("rows", 0) or 0), 1)
-    columns = max(int(profile.get("shape", {}).get("columns", 0) or 0), 1)
+    source_columns = max(int(profile.get("shape", {}).get("columns", 0) or 0), 1)
+    streaming_metadata = profile.get("streaming_metadata", {})
+    profiled_columns = source_columns
+    if isinstance(streaming_metadata, Mapping):
+        profiled_columns = max(
+            int(streaming_metadata.get("profiled_columns", source_columns) or source_columns),
+            1,
+        )
+    column_sampled = profiled_columns < source_columns
+
     missing_counts = profile.get("missing_counts", {})
     missing_total = sum(
         int(value)
         for value in missing_counts.values()
         if isinstance(value, (int, np.integer))
     ) if isinstance(missing_counts, Mapping) else 0
-    missing_percent = missing_total / (rows * columns) * 100
+    missing_percent = missing_total / (rows * profiled_columns) * 100
     duplicate_percent = float(profile.get("duplicate_percent", 0.0) or 0.0)
 
     outlier_percent, sample_outlier_details = calculate_outlier_percent(sample)
@@ -179,18 +187,38 @@ def calculate_health_score_from_profile_sample(
         {
             "method": "streaming_profile_with_bounded_row_sample",
             "source_rows": rows,
-            "source_columns": columns,
+            "source_columns": source_columns,
+            "profiled_columns": profiled_columns,
+            "column_sampled": column_sampled,
             "sample_rows": sample_rows,
-            "sampled": sample_rows < rows,
+            "sampled": sample_rows < rows or column_sampled,
             "full_materialization": False,
             "components": {
-                "completeness": "full_stream_exact",
+                "completeness": (
+                    "full_rows_projected_columns_estimate"
+                    if column_sampled
+                    else "full_stream_exact"
+                ),
                 "uniqueness": (
-                    "full_stream_sample_estimate" if duplicate_estimated else "full_stream_exact"
+                    "projected_columns_row_sample_estimate"
+                    if column_sampled
+                    else (
+                        "full_stream_sample_estimate"
+                        if duplicate_estimated
+                        else "full_stream_exact"
+                    )
+                ),
+                "consistency": (
+                    "full_rows_projected_columns_estimate"
+                    if column_sampled
+                    else "full_stream_profile"
                 ),
-                "consistency": "full_stream_profile",
                 "outlier_safety": (
-                    "bounded_row_sample_estimate" if sample_rows < rows else "exact"
+                    "projected_columns_bounded_row_sample_estimate"
+                    if column_sampled
+                    else (
+                        "bounded_row_sample_estimate" if sample_rows < rows else "exact"
+                    )
                 ),
             },
         },
@@ -203,7 +231,7 @@ def calculate_health_score_from_profile_sample(
         outlier_percent=outlier_percent,
         constant_columns=constant_columns,
         high_cardinality_columns=high_cardinality_columns,
-        columns=columns,
+        columns=profiled_columns,
         outlier_details=outlier_details,
         execution=execution,
     )
diff --git a/src/framevitals/planning_api.py b/src/framevitals/planning_api.py
index 1230c7f..d3063ed 100644
--- a/src/framevitals/planning_api.py
+++ b/src/framevitals/planning_api.py
@@ -8,6 +8,7 @@
 
 from __future__ import annotations
 
+from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
 
@@ -17,7 +18,10 @@
 from framevitals.column_roles import infer_column_roles
 from framevitals.config import ConfigInput, VALID_MODULES, resolve_config
 from framevitals.dataset_signals import detect_dataset_signals
-from framevitals.execution import derive_execution_budget
+from framevitals.execution import (
+    derive_execution_budget,
+    derive_streaming_profile_column_limit,
+)
 from framevitals.planning import AnalysisPlan
 from framevitals.profiler import build_profile
 from framevitals.sources import StreamingDatasetSource, resolve_source
@@ -27,15 +31,64 @@
 PLANNING_SAMPLE_ROWS = 5_000
 
 
+def _evenly_spaced_names(names: Sequence[str], limit: int) -> list[str]:
+    """Select a deterministic schema-wide column projection without NumPy."""
+    total = len(names)
+    if limit < 1:
+        raise ValueError("column projection limit must be at least 1")
+    if total <= limit:
+        return list(names)
+    if limit == 1:
+        return [str(names[0])]
+
+    selected = [
+        str(names[(index * (total - 1)) // (limit - 1)])
+        for index in range(limit)
+    ]
+    return list(dict.fromkeys(selected))
+
+
+def _streaming_projection(
+    source: StreamingDatasetSource,
+    *,
+    source_columns: int,
+    limit: int,
+    target: str | None,
+) -> list[str] | None:
+    """Resolve a bounded projection for ultra-wide streaming planning."""
+    if source_columns <= limit:
+        return None
+
+    schema_method = getattr(source, "schema", None)
+    if not callable(schema_method):
+        raise TypeError(
+            "Ultra-wide streaming planning requires source.schema() so FrameVitals "
+            "can project columns instead of materializing the complete width."
+        )
+    schema = schema_method()
+    names = [str(field.name) for field in schema]
+    if target is not None and target not in names:
+        raise ValueError(f"Target column not found: {target}")
+
+    selected = _evenly_spaced_names(names, limit)
+    if target is not None and target not in selected:
+        if selected:
+            selected[-1] = target
+        else:
+            selected = [target]
+    return list(dict.fromkeys(selected))
+
+
 def _streaming_head_sample(
     source: StreamingDatasetSource,
     *,
     max_rows: int = PLANNING_SAMPLE_ROWS,
+    columns: Sequence[str] | None = None,
 ) -> pd.DataFrame:
     """Read at most ``max_rows`` rows and stop; planning must not scan the file."""
     frames: list[pd.DataFrame] = []
     remaining = int(max_rows)
-    for batch in source.iter_batches(batch_size=max_rows):
+    for batch in source.iter_batches(batch_size=max_rows, columns=columns):
         if remaining <= 0:
             break
         take = min(remaining, int(batch.num_rows))
@@ -55,6 +108,7 @@ def _project_sample_profile_to_source(
     source_rows: int,
     source_columns: int,
     sample_rows: int,
+    profiled_columns: int,
 ) -> dict[str, Any]:
     """Scale rate-based sample metrics to the true source shape for planning."""
     projected = dict(profile)
@@ -73,6 +127,9 @@ def _project_sample_profile_to_source(
         "strategy": "bounded_head",
         "full_scan": False,
         "rate_metrics_projected": sample_rows < source_rows,
+        "profiled_columns": int(profiled_columns),
+        "source_columns": int(source_columns),
+        "column_sampled": int(profiled_columns) < int(source_columns),
     }
     return projected
 
@@ -102,14 +159,33 @@ def plan(
     source_metadata = source.inspect()
 
     if source_metadata.supports_streaming and isinstance(source, StreamingDatasetSource):
-        dataframe = _streaming_head_sample(source)
-        source_rows = int(source_metadata.rows or len(dataframe))
-        source_columns = int(source_metadata.columns or len(dataframe.columns))
+        source_rows = int(source_metadata.rows or 0)
+        source_columns = int(source_metadata.columns or 0)
+        if source_rows < 1 or source_columns < 1:
+            raise ValueError(f"Dataset is empty or has no columns: {source_metadata.name}")
+
+        column_limit = derive_streaming_profile_column_limit(
+            source_rows,
+            source_columns,
+            mode=resolved.mode,
+        )
+        projected_columns = _streaming_projection(
+            source,
+            source_columns=source_columns,
+            limit=column_limit,
+            target=resolved.target,
+        )
+        dataframe = _streaming_head_sample(
+            source,
+            columns=projected_columns,
+        )
+        profiled_columns = int(len(dataframe.columns))
         dataset_profile = _project_sample_profile_to_source(
             build_profile(dataframe),
             source_rows=source_rows,
             source_columns=source_columns,
             sample_rows=len(dataframe),
+            profiled_columns=profiled_columns,
         )
         planning_data = {
             "materialized_full_dataset": False,
@@ -118,6 +194,15 @@ def plan(
             "source_rows": source_rows,
             "strategy": "bounded_head",
             "full_scan": False,
+            "source_columns": source_columns,
+            "sample_columns": profiled_columns,
+            "column_sampled": profiled_columns < source_columns,
+            "column_strategy": (
+                "deterministic_schema_projection"
+                if profiled_columns < source_columns
+                else "full_schema"
+            ),
+            "column_limit": int(column_limit),
         }
     else:
         dataframe = source.load()
@@ -131,6 +216,11 @@ def plan(
             "source_rows": source_rows,
             "strategy": "full_dataset",
             "full_scan": True,
+            "source_columns": source_columns,
+            "sample_columns": source_columns,
+            "column_sampled": False,
+            "column_strategy": "full_schema",
+            "column_limit": source_columns,
         }
 
     source_columns_list = list(dataset_profile.get("columns", dataframe.columns))
diff --git a/src/framevitals/streaming_pipeline.py b/src/framevitals/streaming_pipeline.py
index 1a76fff..b5f62d1 100644
--- a/src/framevitals/streaming_pipeline.py
+++ b/src/framevitals/streaming_pipeline.py
@@ -11,19 +11,22 @@
 from __future__ import annotations
 
 import time
-from collections.abc import Iterable
+from collections.abc import Iterable, Sequence
 from typing import Any
 
 from framevitals.analysis_selector import select_analyses
 from framevitals.column_roles import summarize_roles
 from framevitals.config import VALID_MODULES
 from framevitals.dataset_signals import detect_dataset_signals
-from framevitals.execution import derive_execution_budget
+from framevitals.execution import (
+    derive_execution_budget,
+    derive_streaming_profile_column_limit,
+)
 from framevitals.health_score import calculate_health_score_from_profile_sample
 from framevitals.ml_readiness import calculate_ml_readiness_from_profile
 from framevitals.pipeline import run_full_analysis
 from framevitals.signal_engine import build_signals
-from framevitals.sources import StreamingDatasetSource
+from framevitals.sources import DatasetMetadata, StreamingDatasetSource
 from framevitals.streaming_profile import build_streaming_profile
 from framevitals.streaming_quality import run_streaming_quality_diagnostics
 from framevitals.streaming_roles import infer_streaming_column_roles
@@ -54,6 +57,103 @@ def _working_sample_rows(budget) -> int:
     return min(int(budget.rows), int(requested))
 
 
+def _evenly_spaced_names(names: Sequence[str], limit: int) -> list[str]:
+    total = len(names)
+    if total <= limit:
+        return list(names)
+    if limit <= 1:
+        return [str(names[0])]
+    selected = [
+        str(names[(index * (total - 1)) // (limit - 1)])
+        for index in range(limit)
+    ]
+    return list(dict.fromkeys(selected))
+
+
+def _streaming_profile_projection(
+    source: StreamingDatasetSource,
+    *,
+    source_columns: int,
+    limit: int,
+    target_column: str | None,
+) -> list[str] | None:
+    """Choose a deterministic schema-wide projection for ultra-wide sources."""
+    if source_columns <= limit:
+        return None
+
+    schema_method = getattr(source, "schema", None)
+    if not callable(schema_method):
+        raise TypeError(
+            "Ultra-wide streaming analysis requires source.schema() so FrameVitals "
+            "can project columns instead of scanning the complete width."
+        )
+    schema = schema_method()
+    names = [str(field.name) for field in schema]
+    if target_column is not None and target_column not in names:
+        raise ValueError(f"Target column not found: {target_column}")
+
+    selected = _evenly_spaced_names(names, limit)
+    if target_column is not None and target_column not in selected:
+        selected[-1] = target_column
+    return list(dict.fromkeys(selected))
+
+
+class _ProjectedStreamingSource:
+    """Projection-enforcing view over a streaming source.
+
+    The wrapped source is never asked for the complete ultra-wide width. This
+    keeps the reusable full-row profile scan bounded while preserving source
+    metadata in the outer pipeline.
+    """
+
+    def __init__(self, source: StreamingDatasetSource, columns: Sequence[str]):
+        self._source = source
+        self._columns = tuple(columns)
+
+    def inspect(self) -> DatasetMetadata:
+        metadata = self._source.inspect()
+        return DatasetMetadata(
+            name=metadata.name,
+            kind=metadata.kind,
+            format=metadata.format,
+            rows=metadata.rows,
+            columns=len(self._columns),
+            size_bytes=metadata.size_bytes,
+            materialized=metadata.materialized,
+            supports_projection=True,
+            supports_streaming=True,
+        )
+
+    def schema(self):
+        schema_method = getattr(self._source, "schema", None)
+        if not callable(schema_method):
+            raise TypeError("Projected streaming sources require schema().")
+        schema = schema_method()
+        return [schema.field(name) for name in self._columns]
+
+    def iter_batches(
+        self,
+        *,
+        batch_size: int = 65_536,
+        columns: Sequence[str] | None = None,
+    ):
+        requested = list(self._columns if columns is None else columns)
+        allowed = set(self._columns)
+        unknown = [column for column in requested if column not in allowed]
+        if unknown:
+            raise ValueError(
+                "Projected source requested columns outside its budget: "
+                + ", ".join(unknown[:5])
+            )
+        yield from self._source.iter_batches(
+            batch_size=batch_size,
+            columns=requested,
+        )
+
+    def load(self):
+        raise RuntimeError("Projected streaming source must never materialize fully.")
+
+
 def _scope_bounded_result(
     value: Any,
     *,
@@ -140,15 +240,47 @@ def run_streaming_analysis(
 
     budget = derive_execution_budget(rows, columns, mode=analysis_mode)
     working_rows = _working_sample_rows(budget)
+    profile_column_limit = derive_streaming_profile_column_limit(
+        rows,
+        columns,
+        mode=analysis_mode,
+    )
+    profile_columns = _streaming_profile_projection(
+        source,
+        source_columns=columns,
+        limit=profile_column_limit,
+        target_column=target_column,
+    )
+    profile_source: StreamingDatasetSource = (
+        _ProjectedStreamingSource(source, profile_columns)
+        if profile_columns is not None
+        else source
+    )
 
     profile_start = time.perf_counter()
     profile, working_sample = build_streaming_profile(
-        source,
+        profile_source,
         sample_rows=working_rows,
         return_sample=True,
     )
     streaming_profile_ms = (time.perf_counter() - profile_start) * 1000
 
+    profiled_columns = int(len(working_sample.columns))
+    column_sampled = profiled_columns < columns
+    profile["shape"] = {"rows": rows, "columns": columns}
+    profile["source_metadata"] = metadata.to_dict()
+    streaming_metadata = dict(profile.get("streaming_metadata", {}))
+    streaming_metadata.update({
+        "source_columns": columns,
+        "profiled_columns": profiled_columns,
+        "column_sampled": column_sampled,
+        "column_limit": int(profile_column_limit),
+        "column_strategy": (
+            "deterministic_schema_projection" if column_sampled else "full_schema"
+        ),
+    })
+    profile["streaming_metadata"] = streaming_metadata
+
     if working_sample.empty:
         raise ValueError(f"Streaming source produced no usable rows: {metadata.name}")
     if target_column is not None and target_column not in working_sample.columns:
@@ -197,6 +329,9 @@ def run_streaming_analysis(
         "full_materialization": False,
         "working_sample_rows": int(len(working_sample)),
         "source_rows": rows,
+        "source_columns": columns,
+        "profiled_columns": profiled_columns,
+        "column_sampled": column_sampled,
     }
 
     # Cleaning/charts are internally suppressed on the bounded sample. Charts
@@ -263,15 +398,34 @@ def run_streaming_analysis(
             module_scope[module] = (
                 "bounded_row_sample" if sample_rows < rows else "full_source"
             )
+    profile_scope = (
+        "full_rows_projected_columns" if column_sampled else "full_stream"
+    )
     module_scope.update({
-        "profile": "full_stream",
-        "column_roles": "full_stream_plus_bounded_semantic_sample",
-        "health": "full_stream_plus_bounded_outlier_sample",
-        "ml_readiness": "full_stream_profile",
+        "profile": profile_scope,
+        "column_roles": (
+            "full_rows_projected_columns_plus_bounded_semantic_sample"
+            if column_sampled
+            else "full_stream_plus_bounded_semantic_sample"
+        ),
+        "health": (
+            "full_rows_projected_columns_plus_bounded_outlier_sample"
+            if column_sampled
+            else "full_stream_plus_bounded_outlier_sample"
+        ),
+        "ml_readiness": (
+            "full_rows_projected_columns_profile"
+            if column_sampled
+            else "full_stream_profile"
+        ),
         "quality_diagnostics": (
             "disabled"
             if "quality_diagnostics" in user_disabled
-            else "full_stream_plus_bounded_value_sample"
+            else (
+                "full_rows_projected_columns_plus_bounded_value_sample"
+                if column_sampled
+                else "full_stream_plus_bounded_value_sample"
+            )
         ),
         "cleaning": "deferred_streaming" if "cleaning" not in user_disabled else "disabled",
         "charts": "not_applicable" if "charts" not in user_disabled else "disabled",
@@ -289,6 +443,12 @@ def run_streaming_analysis(
             "full_materialization": False,
             "source_rows": rows,
             "source_columns": columns,
+            "profiled_columns": profiled_columns,
+            "column_sampled": column_sampled,
+            "column_limit": int(profile_column_limit),
+            "column_strategy": (
+                "deterministic_schema_projection" if column_sampled else "full_schema"
+            ),
             "working_sample_rows": sample_rows,
             "working_sample_strategy": sample_strategy,
             "single_full_source_profile_scan": True,
diff --git a/tests/test_extreme_streaming_projection.py b/tests/test_extreme_streaming_projection.py
new file mode 100644
index 0000000..e77add0
--- /dev/null
+++ b/tests/test_extreme_streaming_projection.py
@@ -0,0 +1,107 @@
+from types import SimpleNamespace
+
+import numpy as np
+import pandas as pd
+import pytest
+
+pa = pytest.importorskip("pyarrow")
+
+import framevitals as fv
+from framevitals.execution import derive_streaming_profile_column_limit
+from framevitals.sources import DatasetMetadata
+
+
+class _VirtualSchema:
+    def __init__(self, columns: int):
+        self.columns = int(columns)
+        self.dtype = pa.float64()
+
+    def __iter__(self):
+        for index in range(self.columns):
+            yield SimpleNamespace(name=f"n{index:05d}", type=self.dtype)
+
+    def field(self, name: str):
+        index = int(name[1:])
+        if index < 0 or index >= self.columns:
+            raise KeyError(name)
+        return SimpleNamespace(name=name, type=self.dtype)
+
+
+class _VirtualWideSource:
+    def __init__(self, *, rows: int, columns: int):
+        self.rows = int(rows)
+        self.columns = int(columns)
+        self.max_requested_columns = 0
+        self.unbounded_requested = False
+        self._schema = _VirtualSchema(columns)
+
+    def inspect(self):
+        return DatasetMetadata(
+            name="virtual-wide",
+            kind="virtual",
+            format="synthetic",
+            rows=self.rows,
+            columns=self.columns,
+            size_bytes=self.rows * self.columns * 8,
+            materialized=False,
+            supports_projection=True,
+            supports_streaming=True,
+        )
+
+    def schema(self):
+        return self._schema
+
+    def iter_batches(self, *, batch_size=65_536, columns=None):
+        if columns is None:
+            self.unbounded_requested = True
+            raise AssertionError("ultra-wide source must always be projected")
+        names = list(columns)
+        self.max_requested_columns = max(self.max_requested_columns, len(names))
+        rows = min(int(batch_size), self.rows)
+        values = pa.array(np.arange(rows, dtype=np.float64))
+        yield pa.RecordBatch.from_arrays([values] * len(names), names=names)
+
+    def load(self):
+        raise AssertionError("virtual wide source must never materialize")
+
+
+def test_extreme_deep_streaming_profile_column_limit_is_128():
+    assert (
+        derive_streaming_profile_column_limit(
+            1_000_000,
+            100_000,
+            mode="deep",
+        )
+        == 128
+    )
+
+
+def test_extreme_plan_projects_100k_column_schema():
+    source = _VirtualWideSource(rows=1_000_000, columns=100_000)
+
+    result = fv.plan(source, mode="deep")
+
+    assert result["shape"] == {"rows": 1_000_000, "columns": 100_000}
+    assert result["execution_budget"]["scale_class"] == "extreme"
+    assert result["planning_data"]["column_sampled"] is True
+    assert result["planning_data"]["sample_columns"] == 128
+    assert result["planning_data"]["source_columns"] == 100_000
+    assert source.unbounded_requested is False
+    assert source.max_requested_columns == 128
+
+
+def test_ultra_wide_streaming_analysis_never_requests_full_width():
+    source = _VirtualWideSource(rows=1_000, columns=10_000)
+
+    result = fv.analyze(source, mode="deep", artifacts=False, workers=2)
+
+    streaming = result["execution"]["streaming"]
+    assert streaming["source_rows"] == 1_000
+    assert streaming["source_columns"] == 10_000
+    assert streaming["profiled_columns"] == 128
+    assert streaming["column_sampled"] is True
+    assert streaming["column_strategy"] == "deterministic_schema_projection"
+    assert result["profile"]["shape"] == {"rows": 1_000, "columns": 10_000}
+    assert result["health"]["execution"]["profiled_columns"] == 128
+    assert source.unbounded_requested is False
+    assert source.max_requested_columns == 128

From 159dfb527a7aea91616823620ff157891ee493cd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:28:37 +0530
Subject: [PATCH 384/439] bench: parameterize extreme streaming workload

---
 benchmarks/benchmark_extreme_streaming.py | 75 ++++++++++++++---------
 1 file changed, 45 insertions(+), 30 deletions(-)

diff --git a/benchmarks/benchmark_extreme_streaming.py b/benchmarks/benchmark_extreme_streaming.py
index 24d9ab1..1557575 100644
--- a/benchmarks/benchmark_extreme_streaming.py
+++ b/benchmarks/benchmark_extreme_streaming.py
@@ -1,13 +1,13 @@
-"""Virtual 1,000,000 x 100,000 Deep streaming benchmark.
+"""Virtual ultra-wide streaming benchmark.
 
-A dense float64 materialization of this shape would require roughly 800 GB just
-for raw cells, so the benchmark models a projection-capable streaming source.
-FrameVitals must preserve the true source shape while never requesting the full
-100,000-column width from the source.
+A dense materialization of the target shapes is intentionally avoided. The
+benchmark models a projection-capable streaming source and fails if FrameVitals
+ever requests the complete ultra-wide schema during analysis.
 """
 
 from __future__ import annotations
 
+import argparse
 import json
 from pathlib import Path
 import time
@@ -20,8 +20,9 @@
 from framevitals.sources import DatasetMetadata
 
 
-ROWS = 1_000_000
-COLUMNS = 100_000
+DEFAULT_ROWS = 1_000_000
+DEFAULT_COLUMNS = 100_000
+VALID_MODES = ("quick", "standard", "deep", "research")
 
 
 class VirtualSchema:
@@ -41,7 +42,7 @@ def field(self, name: str):
 
 
 class VirtualExtremeSource:
-    def __init__(self, rows: int = ROWS, columns: int = COLUMNS):
+    def __init__(self, rows: int, columns: int):
         self.rows = int(rows)
         self.columns = int(columns)
         self.schema_view = VirtualSchema(columns)
@@ -52,7 +53,7 @@ def __init__(self, rows: int = ROWS, columns: int = COLUMNS):
 
     def inspect(self):
         return DatasetMetadata(
-            name="virtual-1m-x-100k",
+            name=f"virtual-{self.rows}-x-{self.columns}",
             kind="virtual",
             format="synthetic",
             rows=self.rows,
@@ -70,7 +71,7 @@ def iter_batches(self, *, batch_size=65_536, columns=None):
         if columns is None:
             self.unbounded_requests += 1
             raise RuntimeError(
-                "FrameVitals attempted an unbounded 100,000-column scan."
+                f"FrameVitals attempted an unbounded {self.columns:,}-column scan."
             )
 
         names = list(columns)
@@ -89,38 +90,41 @@ def iter_batches(self, *, batch_size=65_536, columns=None):
     def load(self):
         raise RuntimeError("Virtual extreme source must never materialize fully.")
 
+    def reset_observations(self) -> None:
+        self.max_requested_columns = 0
+        self.unbounded_requests = 0
+        self.batches_yielded = 0
+        self.rows_yielded = 0
 
-def main() -> None:
-    source = VirtualExtremeSource()
+
+def run_benchmark(*, rows: int, columns: int, mode: str) -> dict:
+    source = VirtualExtremeSource(rows, columns)
 
     plan_started = time.perf_counter()
-    plan = fv.plan(source, mode="deep", workers=4)
+    plan = fv.plan(source, mode=mode, workers=4)
     plan_seconds = time.perf_counter() - plan_started
 
-    # Reset counters so execution accounting is independent of planning.
-    source.max_requested_columns = 0
-    source.unbounded_requests = 0
-    source.batches_yielded = 0
-    source.rows_yielded = 0
+    source.reset_observations()
 
     analysis_started = time.perf_counter()
     result = fv.analyze(
         source,
-        mode="deep",
+        mode=mode,
         artifacts=False,
         workers=4,
     )
     analysis_seconds = time.perf_counter() - analysis_started
 
     streaming = result.get("execution", {}).get("streaming", {})
-    payload = {
-        "benchmark_schema_version": 1,
+    working_sample_rows = int(streaming.get("working_sample_rows", 0) or 0)
+    return {
+        "benchmark_schema_version": 2,
         "workload": {
-            "rows": ROWS,
-            "columns": COLUMNS,
-            "cells": ROWS * COLUMNS,
-            "dense_float64_raw_gb": round(ROWS * COLUMNS * 8 / 1_000_000_000, 3),
-            "mode": "deep",
+            "rows": rows,
+            "columns": columns,
+            "cells": rows * columns,
+            "dense_float64_raw_gb": round(rows * columns * 8 / 1_000_000_000, 3),
+            "mode": mode,
             "source": "virtual_projection_capable_stream",
         },
         "plan_seconds": round(plan_seconds, 6),
@@ -140,14 +144,25 @@ def main() -> None:
             "column_sampled": streaming.get("column_sampled"),
             "profiled_columns": streaming.get("profiled_columns"),
             "source_columns": streaming.get("source_columns"),
-            "all_source_rows_scanned": source.rows_yielded == ROWS,
+            "working_sample_rows": working_sample_rows,
+            "all_source_rows_scanned": source.rows_yielded == rows,
             "unbounded_width_requested": source.unbounded_requests > 0,
         },
     }
 
-    output = Path("extreme-benchmark-results.json")
-    output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
-    print(json.dumps(payload, indent=2, sort_keys=True))
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--rows", type=int, default=DEFAULT_ROWS)
+    parser.add_argument("--columns", type=int, default=DEFAULT_COLUMNS)
+    parser.add_argument("--mode", choices=VALID_MODES, default="deep")
+    parser.add_argument("--output", type=Path, default=Path("extreme-benchmark-results.json"))
+    args = parser.parse_args()
+
+    payload = run_benchmark(rows=args.rows, columns=args.columns, mode=args.mode)
+    encoded = json.dumps(payload, indent=2, sort_keys=True)
+    args.output.write_text(encoded + "\n", encoding="utf-8")
+    print(encoded)
 
 
 if __name__ == "__main__":

From 3f8ed05f62935ef87720d9dae39304331aa6bf8f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:28:51 +0530
Subject: [PATCH 385/439] bench: compare all modes on 100k x 100k stream

---
 .../workflows/extreme-streaming-benchmark.yml | 42 ++++++++++++++-----
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/extreme-streaming-benchmark.yml b/.github/workflows/extreme-streaming-benchmark.yml
index 92434fc..de67d78 100644
--- a/.github/workflows/extreme-streaming-benchmark.yml
+++ b/.github/workflows/extreme-streaming-benchmark.yml
@@ -20,9 +20,15 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  deep-1m-x-100k:
-    name: Deep 1M x 100k virtual stream
+  mode-comparison-100k-x-100k:
+    name: ${{ matrix.mode }} 100k x 100k virtual stream
     runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        mode: [quick, standard, deep, research]
 
     steps:
       - name: Checkout repository
@@ -42,33 +48,47 @@ jobs:
           python -m pip install --upgrade pip
           pip install -e ".[arrow]"
 
-      - name: Run 1M x 100k Deep benchmark
+      - name: Run 100k x 100k benchmark
         env:
           FRAMEVITALS_BACKEND: numpy
-        run: python benchmarks/benchmark_extreme_streaming.py
+        run: |
+          python benchmarks/benchmark_extreme_streaming.py \
+            --rows 100000 \
+            --columns 100000 \
+            --mode "${{ matrix.mode }}" \
+            --output "extreme-${{ matrix.mode }}-results.json"
 
       - name: Publish benchmark summary
         if: always()
+        env:
+          MODE: ${{ matrix.mode }}
         run: |
-          if [[ ! -f extreme-benchmark-results.json ]]; then
+          FILE="extreme-${MODE}-results.json"
+          if [[ ! -f "$FILE" ]]; then
+            echo "## ${MODE} 100k x 100k benchmark" >> "$GITHUB_STEP_SUMMARY"
+            echo >> "$GITHUB_STEP_SUMMARY"
+            echo "Benchmark did not produce a result file." >> "$GITHUB_STEP_SUMMARY"
             exit 0
           fi
           python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
           import json
+          import os
           from pathlib import Path
 
-          payload = json.loads(Path("extreme-benchmark-results.json").read_text())
+          mode = os.environ["MODE"]
+          payload = json.loads(Path(f"extreme-{mode}-results.json").read_text())
           workload = payload["workload"]
           safety = payload["safety"]
-          print("## FrameVitals 1M x 100k Deep benchmark")
+          print(f"## FrameVitals {mode.title()} 100k x 100k benchmark")
           print()
           print(f"- Logical cells: **{workload['cells']:,}**")
           print(f"- Dense float64 equivalent: **{workload['dense_float64_raw_gb']:.1f} GB raw**")
           print(f"- Plan: **{payload['plan_seconds']:.3f}s**")
-          print(f"- Deep analysis: **{payload['analysis_seconds']:.3f}s**")
+          print(f"- Analysis: **{payload['analysis_seconds']:.3f}s**")
           print(f"- Scale class: **{payload['scale_class']}**")
           print(f"- Profiled columns: **{safety['profiled_columns']} / {safety['source_columns']}**")
-          print(f"- All 1,000,000 source rows scanned: **{safety['all_source_rows_scanned']}**")
+          print(f"- Working sample rows: **{safety['working_sample_rows']:,}**")
+          print(f"- All 100,000 source rows scanned: **{safety['all_source_rows_scanned']}**")
           print(f"- Full materialization: **{safety['full_materialization']}**")
           print(f"- Unbounded 100k-column request: **{safety['unbounded_width_requested']}**")
           print()
@@ -82,7 +102,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v6
         with:
-          name: framevitals-extreme-1m-x-100k
-          path: extreme-benchmark-results.json
+          name: framevitals-100k-x-100k-${{ matrix.mode }}
+          path: extreme-${{ matrix.mode }}-results.json
           if-no-files-found: ignore
           retention-days: 30

From 76901126dcb4acda5395319ba37c3c7fac43136b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:34:39 +0530
Subject: [PATCH 386/439] bench: add real 100k x 10k parquet benchmark

---
 benchmarks/benchmark_real_wide_parquet.py | 278 ++++++++++++++++++++++
 1 file changed, 278 insertions(+)
 create mode 100644 benchmarks/benchmark_real_wide_parquet.py

diff --git a/benchmarks/benchmark_real_wide_parquet.py b/benchmarks/benchmark_real_wide_parquet.py
new file mode 100644
index 0000000..0797c6c
--- /dev/null
+++ b/benchmarks/benchmark_real_wide_parquet.py
@@ -0,0 +1,278 @@
+"""Generate and benchmark a real 100,000 x 10,000 Parquet dataset.
+
+The dataset is written in bounded row groups, so generation never materializes
+all one billion cells in RAM. The on-disk Parquet file is then analyzed through
+the public ``framevitals.analyze`` API.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import time
+from typing import Any
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+import framevitals as fv
+
+
+DEFAULT_ROWS = 100_000
+DEFAULT_COLUMNS = 10_000
+PATTERN_COUNT = 32
+VALID_MODES = ("quick", "standard", "deep", "research")
+
+
+def _pattern_numpy(pattern_id: int, row_ids: np.ndarray) -> tuple[np.ndarray, np.ndarray | None]:
+    """Return one deterministic int16 pattern and optional null mask."""
+    pattern_id = int(pattern_id) % PATTERN_COUNT
+    if pattern_id == 0:
+        values = row_ids % 101
+    elif pattern_id == 1:
+        values = 2 * (row_ids % 101)
+    elif pattern_id == 2:
+        values = (row_ids * 7 + 3) % 211
+    elif pattern_id == 3:
+        values = np.full(row_ids.shape, 7, dtype=np.int64)
+    elif pattern_id == 4:
+        values = (row_ids * 13) % 503
+        values = values.copy()
+        values[row_ids % 997 == 0] = 30_000
+    elif pattern_id == 5:
+        values = (row_ids * 5) % 307
+        return values.astype(np.int16, copy=False), (row_ids % 10 == 0)
+    elif pattern_id == 6:
+        values = row_ids % 2
+    elif pattern_id == 7:
+        values = np.where(row_ids % 100 < 50, 10, 200)
+    else:
+        multiplier = pattern_id * 2 + 1
+        values = (row_ids * multiplier + pattern_id * 17) % 997
+    return values.astype(np.int16, copy=False), None
+
+
+def _pattern_arrow(pattern_id: int, row_ids: np.ndarray) -> pa.Array:
+    values, mask = _pattern_numpy(pattern_id, row_ids)
+    return pa.array(values, mask=mask, type=pa.int16())
+
+
+def generate_dataset(
+    path: Path,
+    *,
+    rows: int = DEFAULT_ROWS,
+    columns: int = DEFAULT_COLUMNS,
+    row_group_rows: int = 10_000,
+) -> dict[str, Any]:
+    """Write the full logical dataset to Parquet using bounded-memory row groups."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    names = [f"n{index:05d}" for index in range(columns)]
+    schema = pa.schema([pa.field(name, pa.int16()) for name in names])
+
+    started = time.perf_counter()
+    writer = pq.ParquetWriter(
+        path,
+        schema=schema,
+        compression="snappy",
+        use_dictionary=True,
+        write_statistics=False,
+    )
+    try:
+        for start in range(0, rows, row_group_rows):
+            stop = min(rows, start + row_group_rows)
+            row_ids = np.arange(start, stop, dtype=np.int64)
+            patterns = [_pattern_arrow(index, row_ids) for index in range(PATTERN_COUNT)]
+            arrays = [patterns[index % PATTERN_COUNT] for index in range(columns)]
+            batch = pa.RecordBatch.from_arrays(arrays, schema=schema)
+            writer.write_table(pa.Table.from_batches([batch], schema=schema), row_group_size=len(row_ids))
+    finally:
+        writer.close()
+
+    elapsed = time.perf_counter() - started
+    parquet = pq.ParquetFile(path)
+    return {
+        "path": str(path),
+        "rows": int(parquet.metadata.num_rows),
+        "columns": int(parquet.metadata.num_columns),
+        "cells": int(parquet.metadata.num_rows) * int(parquet.metadata.num_columns),
+        "file_size_mb": round(path.stat().st_size / 1_000_000, 3),
+        "dense_int16_raw_gb": round(rows * columns * 2 / 1_000_000_000, 3),
+        "generation_seconds": round(elapsed, 6),
+        "row_groups": int(parquet.metadata.num_row_groups),
+        "pattern_count": PATTERN_COUNT,
+    }
+
+
+def _expected_metrics(rows: int) -> dict[int, dict[str, Any]]:
+    row_ids = np.arange(rows, dtype=np.int64)
+    expected: dict[int, dict[str, Any]] = {}
+    for pattern_id in range(PATTERN_COUNT):
+        values, mask = _pattern_numpy(pattern_id, row_ids)
+        valid = values if mask is None else values[~mask]
+        expected[pattern_id] = {
+            "missing": 0 if mask is None else int(mask.sum()),
+            "count": int(valid.size),
+            "mean": float(valid.astype(np.float64).mean()) if valid.size else None,
+            "min": int(valid.min()) if valid.size else None,
+            "max": int(valid.max()) if valid.size else None,
+        }
+    return expected
+
+
+def _accuracy_checks(result: dict[str, Any], *, rows: int) -> dict[str, Any]:
+    profile = result.get("profile", {})
+    columns = list(profile.get("columns", []))
+    missing_counts = profile.get("missing_counts", {})
+    numeric_summary = profile.get("numeric_summary", {})
+    expected = _expected_metrics(rows)
+
+    missing_matches = 0
+    count_matches = 0
+    minmax_matches = 0
+    mean_errors: list[float] = []
+    expected_missing_total = 0
+
+    for column in columns:
+        index = int(str(column)[1:])
+        truth = expected[index % PATTERN_COUNT]
+        expected_missing_total += int(truth["missing"])
+        if int(missing_counts.get(column, -1)) == int(truth["missing"]):
+            missing_matches += 1
+
+        summary = numeric_summary.get(column, {})
+        if int(summary.get("count", -1)) == int(truth["count"]):
+            count_matches += 1
+        if summary.get("min") == truth["min"] and summary.get("max") == truth["max"]:
+            minmax_matches += 1
+        observed_mean = summary.get("mean")
+        if observed_mean is not None and truth["mean"] is not None:
+            mean_errors.append(abs(float(observed_mean) - float(truth["mean"])))
+
+    expected_health_missing = (
+        expected_missing_total / max(rows * len(columns), 1) * 100.0
+    )
+    observed_health_missing = float(
+        result.get("health", {}).get("details", {}).get("missing_percent", 0.0) or 0.0
+    )
+
+    deep_stats = result.get("deep_statistics_v2", {})
+    numeric_stats = deep_stats.get("numeric_statistics", {}) if isinstance(deep_stats, dict) else {}
+    deep_mean_errors: list[float] = []
+    for column, summary in numeric_stats.items():
+        if not isinstance(summary, dict) or summary.get("mean") is None:
+            continue
+        index = int(str(column)[1:])
+        truth_mean = expected[index % PATTERN_COUNT]["mean"]
+        if truth_mean is not None:
+            deep_mean_errors.append(abs(float(summary["mean"]) - float(truth_mean)))
+
+    timings = result.get("timings_ms", {})
+    phase3 = timings.get("phase3_tasks", {}) if isinstance(timings, dict) else {}
+    return {
+        "profiled_columns_checked": len(columns),
+        "missing_count_exact_matches": missing_matches,
+        "numeric_count_exact_matches": count_matches,
+        "numeric_minmax_exact_matches": minmax_matches,
+        "profile_mean_max_abs_error": round(max(mean_errors), 6) if mean_errors else None,
+        "profile_mean_mean_abs_error": round(float(np.mean(mean_errors)), 6) if mean_errors else None,
+        "expected_projected_missing_percent": round(expected_health_missing, 6),
+        "health_missing_percent": round(observed_health_missing, 6),
+        "health_missing_abs_error": round(abs(observed_health_missing - expected_health_missing), 6),
+        "deep_numeric_columns_checked": len(numeric_stats),
+        "deep_mean_max_abs_error": round(max(deep_mean_errors), 6) if deep_mean_errors else None,
+        "deep_mean_mean_abs_error": round(float(np.mean(deep_mean_errors)), 6) if deep_mean_errors else None,
+        "phase3_tasks_present": sorted(phase3) if isinstance(phase3, dict) else [],
+        "full_pipeline_sections_present": sorted(
+            key
+            for key in (
+                "profile",
+                "column_roles",
+                "health",
+                "ml_readiness",
+                "quality_diagnostics",
+                "advanced",
+                "deep_statistics_v2",
+                "anomalies_v2",
+                "time_series",
+            )
+            if key in result
+        ),
+    }
+
+
+def benchmark_dataset(path: Path, *, mode: str) -> dict[str, Any]:
+    parquet = pq.ParquetFile(path)
+    rows = int(parquet.metadata.num_rows)
+    columns = int(parquet.metadata.num_columns)
+
+    started = time.perf_counter()
+    result = fv.analyze(path, mode=mode, artifacts=False, workers=4)
+    analysis_seconds = time.perf_counter() - started
+
+    streaming = result.get("execution", {}).get("streaming", {})
+    return {
+        "benchmark_schema_version": 1,
+        "workload": {
+            "rows": rows,
+            "columns": columns,
+            "cells": rows * columns,
+            "file_size_mb": round(path.stat().st_size / 1_000_000, 3),
+            "dense_int16_raw_gb": round(rows * columns * 2 / 1_000_000_000, 3),
+            "mode": mode,
+            "source": "real_parquet_file",
+        },
+        "analysis_seconds": round(analysis_seconds, 6),
+        "pipeline_timings_ms": result.get("timings_ms", {}),
+        "streaming_execution": streaming,
+        "accuracy": _accuracy_checks(result, rows=rows),
+        "safety": {
+            "full_materialization": streaming.get("full_materialization"),
+            "source_rows": streaming.get("source_rows"),
+            "source_columns": streaming.get("source_columns"),
+            "profiled_columns": streaming.get("profiled_columns"),
+            "working_sample_rows": streaming.get("working_sample_rows"),
+            "column_sampled": streaming.get("column_sampled"),
+        },
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dataset", type=Path, default=Path("wide-100k-x-10k.parquet"))
+    parser.add_argument("--rows", type=int, default=DEFAULT_ROWS)
+    parser.add_argument("--columns", type=int, default=DEFAULT_COLUMNS)
+    parser.add_argument("--row-group-rows", type=int, default=10_000)
+    parser.add_argument("--mode", choices=VALID_MODES)
+    parser.add_argument("--generate-only", action="store_true")
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+
+    if args.generate_only:
+        payload = generate_dataset(
+            args.dataset,
+            rows=args.rows,
+            columns=args.columns,
+            row_group_rows=args.row_group_rows,
+        )
+    else:
+        if not args.dataset.exists():
+            generate_dataset(
+                args.dataset,
+                rows=args.rows,
+                columns=args.columns,
+                row_group_rows=args.row_group_rows,
+            )
+        if args.mode is None:
+            parser.error("--mode is required unless --generate-only is used")
+        payload = benchmark_dataset(args.dataset, mode=args.mode)
+
+    encoded = json.dumps(payload, indent=2, sort_keys=True)
+    if args.output is not None:
+        args.output.write_text(encoded + "\n", encoding="utf-8")
+    print(encoded)
+
+
+if __name__ == "__main__":
+    main()

From 4d6c5ed384278335c5c74753f7e0fc33240d90c2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:34:55 +0530
Subject: [PATCH 387/439] ci: run real 100k x 10k parquet analysis benchmark

---
 .../workflows/real-wide-parquet-benchmark.yml | 160 ++++++++++++++++++
 1 file changed, 160 insertions(+)
 create mode 100644 .github/workflows/real-wide-parquet-benchmark.yml

diff --git a/.github/workflows/real-wide-parquet-benchmark.yml b/.github/workflows/real-wide-parquet-benchmark.yml
new file mode 100644
index 0000000..461f4de
--- /dev/null
+++ b/.github/workflows/real-wide-parquet-benchmark.yml
@@ -0,0 +1,160 @@
+name: Real Wide Parquet Benchmark
+
+on:
+  push:
+    branches: ["develop/august"]
+    paths:
+      - "benchmarks/benchmark_real_wide_parquet.py"
+      - ".github/workflows/real-wide-parquet-benchmark.yml"
+      - "src/framevitals/**"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark:
+    name: Full pipeline 100k x 10k real Parquet
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install benchmark runtime
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[arrow]"
+
+      - name: Generate real 100k x 10k Parquet dataset
+        run: |
+          python benchmarks/benchmark_real_wide_parquet.py \
+            --generate-only \
+            --dataset wide-100k-x-10k.parquet \
+            --rows 100000 \
+            --columns 10000 \
+            --row-group-rows 10000 \
+            --output generation.json
+
+      - name: Run all four full analysis modes
+        env:
+          FRAMEVITALS_BACKEND: numpy
+        run: |
+          for mode in quick standard deep research; do
+            echo "=== ${mode} ==="
+            python benchmarks/benchmark_real_wide_parquet.py \
+              --dataset wide-100k-x-10k.parquet \
+              --mode "${mode}" \
+              --output "real-wide-${mode}.json"
+          done
+
+      - name: Validate full-pipeline accuracy
+        run: |
+          python - <<'PY'
+          import json
+          from pathlib import Path
+
+          failures = []
+          for mode in ("quick", "standard", "deep", "research"):
+              payload = json.loads(Path(f"real-wide-{mode}.json").read_text())
+              accuracy = payload["accuracy"]
+              checked = accuracy["profiled_columns_checked"]
+              if accuracy["missing_count_exact_matches"] != checked:
+                  failures.append(f"{mode}: missing-count parity failed")
+              if accuracy["numeric_count_exact_matches"] != checked:
+                  failures.append(f"{mode}: numeric-count parity failed")
+              if accuracy["numeric_minmax_exact_matches"] != checked:
+                  failures.append(f"{mode}: min/max parity failed")
+              if (accuracy["profile_mean_max_abs_error"] or 0.0) > 0.01:
+                  failures.append(
+                      f"{mode}: profile mean max error {accuracy['profile_mean_max_abs_error']}"
+                  )
+              if accuracy["health_missing_abs_error"] > 0.02:
+                  failures.append(
+                      f"{mode}: health missingness error {accuracy['health_missing_abs_error']}"
+                  )
+              if (accuracy["deep_mean_max_abs_error"] or 0.0) > 8.0:
+                  failures.append(
+                      f"{mode}: bounded deep-stat mean error {accuracy['deep_mean_max_abs_error']}"
+                  )
+              if payload["safety"]["full_materialization"] is not False:
+                  failures.append(f"{mode}: full materialization occurred")
+
+          if failures:
+              raise SystemExit("\n".join(failures))
+          print("All full-pipeline accuracy checks passed.")
+          PY
+
+      - name: Publish benchmark summary
+        if: always()
+        run: |
+          python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
+          import json
+          from pathlib import Path
+
+          generation_path = Path("generation.json")
+          if generation_path.exists():
+              generation = json.loads(generation_path.read_text())
+              print("## Real 100k x 10k Parquet dataset")
+              print()
+              print(f"- Logical cells: **{generation['cells']:,}**")
+              print(f"- Dense int16 equivalent: **{generation['dense_int16_raw_gb']:.2f} GB**")
+              print(f"- Parquet file size: **{generation['file_size_mb']:.2f} MB**")
+              print(f"- Generation time: **{generation['generation_seconds']:.3f}s**")
+              print(f"- Row groups: **{generation['row_groups']}**")
+              print()
+
+          print("## Full analysis results")
+          print()
+          print("| Mode | Wall time | Profiled cols | Sample rows | Profile mean max error | Deep mean max error | Health missing error |")
+          print("|---|---:|---:|---:|---:|---:|---:|")
+          for mode in ("quick", "standard", "deep", "research"):
+              path = Path(f"real-wide-{mode}.json")
+              if not path.exists():
+                  print(f"| {mode.title()} | failed | - | - | - | - | - |")
+                  continue
+              payload = json.loads(path.read_text())
+              safety = payload["safety"]
+              accuracy = payload["accuracy"]
+              print(
+                  f"| {mode.title()} | {payload['analysis_seconds']:.3f}s | "
+                  f"{safety['profiled_columns']} | {safety['working_sample_rows']} | "
+                  f"{accuracy['profile_mean_max_abs_error']} | "
+                  f"{accuracy['deep_mean_max_abs_error']} | "
+                  f"{accuracy['health_missing_abs_error']} |"
+              )
+              print()
+              print(f"### {mode.title()} pipeline stages")
+              for name, milliseconds in sorted(payload.get("pipeline_timings_ms", {}).items()):
+                  if isinstance(milliseconds, (int, float)):
+                      print(f"- {name}: {milliseconds / 1000:.3f}s")
+              print()
+          PY
+
+      - name: Upload benchmark results
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: framevitals-real-wide-100k-x-10k
+          path: |
+            generation.json
+            real-wide-quick.json
+            real-wide-standard.json
+            real-wide-deep.json
+            real-wide-research.json
+          if-no-files-found: ignore
+          retention-days: 30

From a061718a62a46c49678003c3439c949495168fc6 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:42:58 +0530
Subject: [PATCH 388/439] fix: scope ML readiness to profiled streaming columns

---
 src/framevitals/ml_readiness.py | 62 +++++++++++++++++++++++++++------
 1 file changed, 52 insertions(+), 10 deletions(-)

diff --git a/src/framevitals/ml_readiness.py b/src/framevitals/ml_readiness.py
index 2d97403..7feed77 100644
--- a/src/framevitals/ml_readiness.py
+++ b/src/framevitals/ml_readiness.py
@@ -61,10 +61,27 @@ def _score_ml_readiness(
 
 
 def calculate_ml_readiness_from_profile(profile: dict[str, Any]) -> dict[str, Any]:
-    """Calculate ML-readiness directly from a completed dataset profile."""
+    """Calculate ML-readiness directly from a completed dataset profile.
+
+    Streaming ultra-wide profiles may intentionally cover all source rows but
+    only a deterministic projection of columns. In that case missingness and
+    column-group penalties must be normalized by the profiled width, not by the
+    unobserved source width. The result is explicitly marked as a projected-
+    column estimate rather than a full-schema exact score.
+    """
     shape = profile.get("shape", {})
     rows = int(shape.get("rows", 0) or 0)
-    columns = int(shape.get("columns", len(profile.get("columns", []))) or 0)
+    source_columns = int(shape.get("columns", len(profile.get("columns", []))) or 0)
+    streaming = profile.get("streaming_metadata", {})
+    profiled_columns = int(
+        streaming.get("profiled_columns", len(profile.get("columns", [])))
+        or len(profile.get("columns", []))
+    )
+    column_sampled = bool(streaming.get("column_sampled", False)) or (
+        profiled_columns < source_columns
+    )
+    scoring_columns = profiled_columns if column_sampled else source_columns
+
     missing_total = sum(
         int(value)
         for value in profile.get("missing_counts", {}).values()
@@ -76,31 +93,56 @@ def calculate_ml_readiness_from_profile(profile: dict[str, Any]) -> dict[str, An
 
     result = _score_ml_readiness(
         rows=rows,
-        columns=columns,
+        columns=scoring_columns,
         missing_total=missing_total,
         duplicate_percent=duplicate_percent,
         categorical_cols=categorical_cols,
         numeric_cols=numeric_cols,
     )
 
-    streaming = profile.get("streaming_metadata", {})
+    if column_sampled:
+        result["score_scope"] = "full_rows_projected_columns_estimate"
+        result["profiled_columns"] = profiled_columns
+        result["source_columns"] = source_columns
+        result["issues"]["missing_percent_scope"] = "profiled_columns"
+        result["issues"]["duplicate_percent_scope"] = (
+            "bounded_rows_projected_columns"
+        )
+    else:
+        result["score_scope"] = "full_schema"
+
     if streaming.get("enabled"):
         duplicate_metadata = profile.get("duplicate_metadata", {})
         duplicate_sampled = bool(duplicate_metadata.get("sampled"))
+        if column_sampled:
+            missingness_scope = "full_rows_projected_columns_exact"
+            column_group_scope = "projected_schema_exact"
+            duplicate_scope = (
+                "bounded_row_sample_projected_columns_estimate"
+                if duplicate_sampled
+                else "projected_columns_exact"
+            )
+        else:
+            missingness_scope = "full_stream_exact"
+            column_group_scope = "schema_exact"
+            duplicate_scope = (
+                "bounded_row_sample_estimate" if duplicate_sampled else "exact"
+            )
+
         result["execution"] = normalize_execution(
             {
                 "method": "streaming_profile",
                 "full_materialization": bool(streaming.get("full_materialization", False)),
                 "source_rows": rows,
-                "source_columns": columns,
+                "source_columns": source_columns,
+                "profiled_columns": profiled_columns,
+                "column_sampled": column_sampled,
                 "sample_rows": int(streaming.get("sample_rows", 0) or 0),
                 "sampled": int(streaming.get("sample_rows", 0) or 0) < rows,
                 "components": {
-                    "missingness": "full_stream_exact",
-                    "column_groups": "schema_exact",
-                    "duplicate_rate": (
-                        "bounded_row_sample_estimate" if duplicate_sampled else "exact"
-                    ),
+                    "missingness": missingness_scope,
+                    "column_groups": column_group_scope,
+                    "duplicate_rate": duplicate_scope,
                 },
             },
             method="streaming_profile",

From a1135bd21820e46821e82db370c9f74c2695ee5f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:43:14 +0530
Subject: [PATCH 389/439] fix: report projected quality coverage accurately

---
 src/framevitals/streaming_quality.py | 46 +++++++++++++++++++++++-----
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/src/framevitals/streaming_quality.py b/src/framevitals/streaming_quality.py
index 77fc626..a1d641e 100644
--- a/src/framevitals/streaming_quality.py
+++ b/src/framevitals/streaming_quality.py
@@ -93,10 +93,10 @@ def run_streaming_quality_diagnostics(
 ) -> dict[str, Any]:
     """Run quality diagnostics without materializing a streaming source.
 
-    Full-source missingness and duplicate-row estimates come from ``profile``.
-    Value-level diagnostics operate on ``sample``. The returned execution
-    metadata describes that split so downstream consumers can distinguish
-    observed facts from sample-derived candidates.
+    Full-row profile facts may cover either the complete schema or a projected
+    ultra-wide schema. Value-level diagnostics operate on ``sample``. Execution
+    metadata records both row and column coverage so projected checks cannot be
+    mistaken for full-schema facts.
     """
     if source_rows < 1:
         raise ValueError("source_rows must be at least 1.")
@@ -114,16 +114,29 @@ def run_streaming_quality_diagnostics(
     )
 
     sample_rows = int(len(sample))
+    profiled_columns = int(
+        profile.get("streaming_metadata", {}).get(
+            "profiled_columns",
+            len(profile.get("columns", sample.columns)),
+        )
+        or len(profile.get("columns", sample.columns))
+    )
+    column_sampled = bool(
+        profile.get("streaming_metadata", {}).get("column_sampled", False)
+    ) or profiled_columns < int(source_columns)
+
     _annotate_sample_findings(
         payload,
         source_rows=int(source_rows),
         sample_rows=sample_rows,
     )
 
+    columns_checked = min(int(len(sample.columns)), int(max_columns))
     payload["rows"] = int(source_rows)
     payload["columns"] = int(source_columns)
-    payload["columns_checked"] = min(int(source_columns), int(max_columns))
-    payload["truncated_columns"] = int(source_columns) > int(max_columns)
+    payload["profiled_columns"] = profiled_columns
+    payload["columns_checked"] = columns_checked
+    payload["truncated_columns"] = columns_checked < int(source_columns)
     payload["duplicate_rows"] = int(profile.get("duplicate_rows", 0) or 0)
 
     issue_groups = (
@@ -148,15 +161,32 @@ def run_streaming_quality_diagnostics(
         "issue_count": issue_count + (1 if duplicate_rows else 0),
         "primary_key_candidate_count": len(payload.get("primary_key_candidates", [])),
     }
+
+    profile_fact_scope = (
+        "full_rows_projected_columns" if column_sampled else "full_stream"
+    )
     payload["execution"] = normalize_execution(
         {
             "method": "streaming_profile_with_bounded_quality_sample",
             "full_materialization": False,
             "source_rows": int(source_rows),
             "source_columns": int(source_columns),
+            "profiled_columns": profiled_columns,
+            "columns_checked": columns_checked,
+            "column_sampled": column_sampled,
             "sample_rows": sample_rows,
             "sampled": sample_rows < int(source_rows),
-            "full_source_inputs": ["missingness", "duplicate_row_estimate"],
+            "profile_fact_scope": profile_fact_scope,
+            "full_source_inputs": (
+                ["missingness", "duplicate_row_estimate"]
+                if not column_sampled
+                else []
+            ),
+            "projected_full_row_inputs": (
+                ["missingness", "duplicate_row_estimate"]
+                if column_sampled
+                else []
+            ),
             "sample_inputs": [
                 "column_roles",
                 "identifier_duplicates",
@@ -171,7 +201,7 @@ def run_streaming_quality_diagnostics(
             ],
             "candidate_only_checks": (
                 ["primary_key_candidates", "duplicate_columns"]
-                if sample_rows < int(source_rows)
+                if sample_rows < int(source_rows) or column_sampled
                 else []
             ),
         },

From 0e25d561a038386715dc8db625bf077550d70eee Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:43:25 +0530
Subject: [PATCH 390/439] test: cover projected streaming scoring semantics

---
 tests/test_projected_streaming_semantics.py | 72 +++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 tests/test_projected_streaming_semantics.py

diff --git a/tests/test_projected_streaming_semantics.py b/tests/test_projected_streaming_semantics.py
new file mode 100644
index 0000000..329b0e3
--- /dev/null
+++ b/tests/test_projected_streaming_semantics.py
@@ -0,0 +1,72 @@
+import pandas as pd
+import pytest
+
+from framevitals.ml_readiness import calculate_ml_readiness_from_profile
+from framevitals.streaming_quality import run_streaming_quality_diagnostics
+
+
+def _projected_profile(*, rows: int = 100, source_columns: int = 10_000, profiled_columns: int = 64):
+    columns = [f"c{i}" for i in range(profiled_columns)]
+    return {
+        "shape": {"rows": rows, "columns": source_columns},
+        "columns": columns,
+        "numeric_columns": columns,
+        "categorical_columns": [],
+        "missing_counts": {name: rows for name in columns},
+        "missing_percent": {name: 100.0 for name in columns},
+        "duplicate_rows": 0,
+        "duplicate_percent": 0.0,
+        "duplicate_metadata": {"sampled": True},
+        "streaming_metadata": {
+            "enabled": True,
+            "full_materialization": False,
+            "sample_rows": min(rows, 50),
+            "source_columns": source_columns,
+            "profiled_columns": profiled_columns,
+            "column_sampled": True,
+        },
+    }
+
+
+def test_ml_readiness_uses_profiled_width_for_projected_missingness():
+    profile = _projected_profile()
+    result = calculate_ml_readiness_from_profile(profile)
+
+    assert result["issues"]["missing_percent"] == pytest.approx(100.0)
+    assert result["score_scope"] == "full_rows_projected_columns_estimate"
+    assert result["profiled_columns"] == 64
+    assert result["source_columns"] == 10_000
+    assert result["execution"]["components"]["missingness"] == (
+        "full_rows_projected_columns_exact"
+    )
+    assert result["execution"]["components"]["column_groups"] == "projected_schema_exact"
+    assert result["execution"]["components"]["duplicate_rate"] == (
+        "bounded_row_sample_projected_columns_estimate"
+    )
+
+
+def test_streaming_quality_reports_only_columns_actually_checked():
+    profile = _projected_profile(rows=200, profiled_columns=64)
+    sample = pd.DataFrame({f"c{i}": [i, i + 1, i + 2] for i in range(64)})
+
+    result = run_streaming_quality_diagnostics(
+        sample,
+        profile=profile,
+        source_rows=200,
+        source_columns=10_000,
+        max_sample_rows=3,
+        max_columns=100,
+    )
+
+    assert result["columns"] == 10_000
+    assert result["profiled_columns"] == 64
+    assert result["columns_checked"] == 64
+    assert result["truncated_columns"] is True
+    execution = result["execution"]
+    assert execution["column_sampled"] is True
+    assert execution["profile_fact_scope"] == "full_rows_projected_columns"
+    assert execution["full_source_inputs"] == []
+    assert set(execution["projected_full_row_inputs"]) == {
+        "missingness",
+        "duplicate_row_estimate",
+    }

From 494952bf5d59fdf708d4ee37d66eadc3468d7719 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:43:52 +0530
Subject: [PATCH 391/439] perf: adapt research confidence intervals by sample
 size

---
 src/framevitals/budgeted_analysis.py | 36 +++++++++++++++++++++-------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/src/framevitals/budgeted_analysis.py b/src/framevitals/budgeted_analysis.py
index 8ed850f..d7c8c61 100644
--- a/src/framevitals/budgeted_analysis.py
+++ b/src/framevitals/budgeted_analysis.py
@@ -13,16 +13,19 @@
 
 from framevitals.anomaly_ensemble import detect_anomalies_ensemble
 from framevitals.deep_statistics_v2 import run_deep_statistics_v2
-from framevitals.fast_deep_statistics import run_fast_deep_statistics_v2
 from framevitals.deep_triage import triage_deep_columns
 from framevitals.execution import ExecutionBudget, deterministic_sample_frame
 from framevitals.fast_anomaly import fast_anomaly_scan
+from framevitals.fast_deep_statistics import run_fast_deep_statistics_v2
 from framevitals.neural_anomaly import neural_reconstruction_anomalies
 from framevitals.provenance import normalize_execution
 from framevitals.stream_change import scan_ordered_mean_shift
 from framevitals.time_series import detect_and_analyze_time_series
 
 
+_RESEARCH_BCA_MAX_ROWS = 2_000
+
+
 def _attach_execution(
     payload: dict[str, Any],
     *,
@@ -49,7 +52,14 @@ def run_budgeted_deep_statistics(
     budget: ExecutionBudget,
     max_pairs: int | None = None,
 ) -> dict[str, Any]:
-    """Run deep statistics on a bounded, adaptively selected diagnostic view."""
+    """Run deep statistics on a bounded, adaptively selected diagnostic view.
+
+    Research mode keeps BCa bootstrap semantics for genuinely small diagnostic
+    samples, where resampling cost is bounded and finite-sample refinement can
+    be useful. Once the diagnostic sample is larger, deterministic Student-t
+    mean intervals and distribution-free median order-statistic intervals are
+    used instead. The rest of the Research statistical battery is unchanged.
+    """
     sample_limit = max(
         20,
         min(
@@ -76,24 +86,33 @@ def run_budgeted_deep_statistics(
     if pair_budget < 1:
         raise ValueError("max_pairs must be at least 1.")
 
-    if budget.mode == "research":
+    use_research_bca = (
+        budget.mode == "research" and len(diagnostic_view) <= _RESEARCH_BCA_MAX_ROWS
+    )
+    if use_research_bca:
         payload = run_deep_statistics_v2(diagnostic_view, max_pairs=pair_budget)
-        inference_strategy = "bca_bootstrap"
+        inference_strategy = "bca_bootstrap_small_sample"
     else:
         payload = run_fast_deep_statistics_v2(
             diagnostic_view,
             max_pairs=pair_budget,
         )
-        inference_strategy = "closed_form_and_order_statistics"
+        inference_strategy = (
+            "adaptive_large_sample_closed_form_and_order_statistics"
+            if budget.mode == "research"
+            else "closed_form_and_order_statistics"
+        )
+
     triage_payload = triage.to_dict()
     payload["column_triage"] = triage_payload
 
     sampling = {
         **sampling,
         "reason": (
-            "Deep statistics use a bounded row view plus adaptive column triage; "
-            "expensive distribution work is reserved for the highest-interest columns; "
-            "non-research modes use O(n) confidence intervals instead of resampling."
+            "Deep statistics use a bounded row view plus adaptive column triage. "
+            "Research mode retains BCa bootstrap on small diagnostic samples but "
+            "switches to deterministic large-sample intervals above the BCa cost "
+            "threshold; other modes use the deterministic interval path directly."
         ),
         "pair_budget": int(pair_budget),
         "column_triage": triage_payload,
@@ -101,6 +120,7 @@ def run_budgeted_deep_statistics(
         "diagnostic_columns": int(len(selected_columns)),
         "adaptive_strategy": "column_interest_triage",
         "inference_strategy": inference_strategy,
+        "research_bca_max_rows": int(_RESEARCH_BCA_MAX_ROWS),
     }
     return _attach_execution(
         payload,

From 852d26b0336e5a44a32aa30bfccb806c83a53aad Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:45:35 +0530
Subject: [PATCH 392/439] test: enforce adaptive research inference policy

---
 tests/test_adaptive_execution.py | 56 +++++++++++++++++++++++++++++---
 1 file changed, 51 insertions(+), 5 deletions(-)

diff --git a/tests/test_adaptive_execution.py b/tests/test_adaptive_execution.py
index 97346f2..31d01a8 100644
--- a/tests/test_adaptive_execution.py
+++ b/tests/test_adaptive_execution.py
@@ -102,26 +102,72 @@ def fake_deep(frame, max_pairs=20):
     )
 
 
-def test_research_deep_statistics_adapter_keeps_bca(monkeypatch):
-    seen = {}
+def test_research_deep_statistics_adapter_keeps_bca_for_small_samples(monkeypatch):
+    seen = {"bca": 0, "fast": 0}
 
-    def fake_deep(frame, max_pairs=20):
+    def fake_bca(frame, max_pairs=20):
+        seen["bca"] += 1
         seen["rows"] = len(frame)
         seen["pairs"] = max_pairs
         return {"available": True}
 
+    def fake_fast(frame, max_pairs=20):
+        seen["fast"] += 1
+        return {"available": True}
+
     monkeypatch.setattr(
         "framevitals.budgeted_analysis.run_deep_statistics_v2",
-        fake_deep,
+        fake_bca,
+    )
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.run_fast_deep_statistics_v2",
+        fake_fast,
     )
     frame = pd.DataFrame({"x": np.arange(500), "y": np.arange(500)})
     budget = derive_execution_budget(len(frame), len(frame.columns), mode="research")
 
     result = run_budgeted_deep_statistics(frame, budget=budget)
 
+    assert seen["bca"] == 1
+    assert seen["fast"] == 0
+    assert seen["rows"] == len(frame)
+    assert seen["pairs"] <= budget.relationship_pair_budget
+    assert result["execution"]["inference_strategy"] == "bca_bootstrap_small_sample"
+
+
+def test_research_deep_statistics_adapter_uses_fast_intervals_for_large_samples(monkeypatch):
+    seen = {"bca": 0, "fast": 0}
+
+    def fake_bca(frame, max_pairs=20):
+        seen["bca"] += 1
+        return {"available": True}
+
+    def fake_fast(frame, max_pairs=20):
+        seen["fast"] += 1
+        seen["rows"] = len(frame)
+        seen["pairs"] = max_pairs
+        return {"available": True}
+
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.run_deep_statistics_v2",
+        fake_bca,
+    )
+    monkeypatch.setattr(
+        "framevitals.budgeted_analysis.run_fast_deep_statistics_v2",
+        fake_fast,
+    )
+    frame = pd.DataFrame({"x": np.arange(5_000), "y": np.arange(5_000)})
+    budget = derive_execution_budget(len(frame), len(frame.columns), mode="research")
+
+    result = run_budgeted_deep_statistics(frame, budget=budget)
+
+    assert seen["bca"] == 0
+    assert seen["fast"] == 1
     assert seen["rows"] == len(frame)
     assert seen["pairs"] <= budget.relationship_pair_budget
-    assert result["execution"]["inference_strategy"] == "bca_bootstrap"
+    assert result["execution"]["inference_strategy"] == (
+        "adaptive_large_sample_closed_form_and_order_statistics"
+    )
 
 
 def test_anomaly_adapter_discloses_sample_coverage(monkeypatch):

From fd3c2472c1bfd1a96253c28183165ed9342962f7 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:46:23 +0530
Subject: [PATCH 393/439] feat: reuse exact streaming statistics downstream

---
 src/framevitals/streaming_exact_reuse.py | 96 ++++++++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 src/framevitals/streaming_exact_reuse.py

diff --git a/src/framevitals/streaming_exact_reuse.py b/src/framevitals/streaming_exact_reuse.py
new file mode 100644
index 0000000..acf5a48
--- /dev/null
+++ b/src/framevitals/streaming_exact_reuse.py
@@ -0,0 +1,96 @@
+"""Reuse full-stream sufficient statistics in bounded downstream analyses.
+
+The streaming profiler has already scanned every selected source row and owns
+higher-quality sufficient statistics than any bounded diagnostic sample can
+reconstruct. This module enforces FrameVitals' exact-once rule: downstream
+analysis payloads may add sample-dependent diagnostics, but they must not replace
+known full-stream moments with noisier sample estimates.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+_EXACT_NUMERIC_FIELD_MAP = {
+    "count": "count",
+    "mean": "mean",
+    "std": "std",
+    "min": "min",
+    "max": "max",
+}
+
+
+def reuse_streaming_exact_statistics(payload: dict[str, Any]) -> dict[str, Any]:
+    """Overlay full-stream numeric sufficient statistics onto deep diagnostics.
+
+    Quantiles, skew/kurtosis, outlier views, distribution fits, hypothesis tests,
+    confidence intervals, and bivariate tests remain bounded-sample diagnostics.
+    Only statistics that the streaming profile already computed over every
+    profiled source row are reused.
+    """
+    profile = payload.get("profile")
+    deep = payload.get("deep_statistics_v2")
+    if not isinstance(profile, dict) or not isinstance(deep, dict):
+        return payload
+
+    deep_numeric = deep.get("numeric_statistics")
+    full_numeric = profile.get("numeric_summary")
+    if not isinstance(deep_numeric, dict) or not isinstance(full_numeric, dict):
+        return payload
+
+    streaming = profile.get("streaming_metadata", {})
+    backend = streaming.get("numeric_backend")
+    profile_metadata = profile.get("numeric_summary_metadata", {})
+    method = profile_metadata.get("method") if isinstance(profile_metadata, dict) else None
+
+    reused_columns = 0
+    for column, sample_summary in deep_numeric.items():
+        if not isinstance(sample_summary, dict):
+            continue
+        exact_summary = full_numeric.get(column)
+        if not isinstance(exact_summary, dict):
+            continue
+
+        reused_fields: list[str] = []
+        for target_field, source_field in _EXACT_NUMERIC_FIELD_MAP.items():
+            if source_field not in exact_summary:
+                continue
+            sample_summary[target_field] = exact_summary[source_field]
+            reused_fields.append(target_field)
+
+        if reused_fields:
+            reused_columns += 1
+            sample_summary["summary_provenance"] = {
+                "scope": "full_stream",
+                "backend": backend,
+                "method": method,
+                "reused_exact_fields": reused_fields,
+                "sample_derived_fields": [
+                    "q1",
+                    "median",
+                    "q3",
+                    "iqr",
+                    "skewness",
+                    "kurtosis",
+                    "outliers",
+                    "normality",
+                    "distribution_fit",
+                    "bootstrap_mean_ci",
+                    "bootstrap_median_ci",
+                ],
+            }
+
+    if reused_columns:
+        execution = deep.setdefault("execution", {})
+        if isinstance(execution, dict):
+            execution["exact_once_reuse"] = {
+                "enabled": True,
+                "source": "streaming_profile",
+                "backend": backend,
+                "method": method,
+                "columns_reused": reused_columns,
+                "fields": list(_EXACT_NUMERIC_FIELD_MAP),
+            }
+
+    return payload

From d331294a5e5863ae39f47957019b94349b09b8f3 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:46:37 +0530
Subject: [PATCH 394/439] fix: enforce exact-once streaming summaries

---
 src/framevitals/analysis_api.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/framevitals/analysis_api.py b/src/framevitals/analysis_api.py
index 90e0521..9e0b5e7 100644
--- a/src/framevitals/analysis_api.py
+++ b/src/framevitals/analysis_api.py
@@ -117,6 +117,7 @@ def analyze(
             and isinstance(source, StreamingDatasetSource)
             and not resolved.artifacts
         ):
+            from framevitals.streaming_exact_reuse import reuse_streaming_exact_statistics
             from framevitals.streaming_pipeline import run_streaming_analysis
 
             payload = run_streaming_analysis(
@@ -129,6 +130,7 @@ def analyze(
                 skip_ai=True,
                 disabled_modules=effective_disabled,
             )
+            payload = reuse_streaming_exact_statistics(payload)
         elif isinstance(data, (str, Path)):
             path = Path(data)
             if not path.exists():

From 2b564862efb3238579044bac09aa8fc900a18dfc Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:46:46 +0530
Subject: [PATCH 395/439] test: protect streaming exact-once reuse

---
 tests/test_streaming_exact_reuse.py | 78 +++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100644 tests/test_streaming_exact_reuse.py

diff --git a/tests/test_streaming_exact_reuse.py b/tests/test_streaming_exact_reuse.py
new file mode 100644
index 0000000..64623bb
--- /dev/null
+++ b/tests/test_streaming_exact_reuse.py
@@ -0,0 +1,78 @@
+from framevitals.streaming_exact_reuse import reuse_streaming_exact_statistics
+
+
+def test_reuses_full_stream_moments_without_overwriting_sample_diagnostics():
+    payload = {
+        "profile": {
+            "numeric_summary": {
+                "x": {
+                    "count": 100,
+                    "mean": 12.5,
+                    "std": 3.25,
+                    "min": -2.0,
+                    "25%": 10.0,
+                    "50%": 12.0,
+                    "75%": 15.0,
+                    "max": 30.0,
+                }
+            },
+            "numeric_summary_metadata": {
+                "backend": "rust",
+                "method": "native_streaming_accumulator",
+            },
+            "streaming_metadata": {
+                "enabled": True,
+                "numeric_backend": "rust",
+            },
+        },
+        "deep_statistics_v2": {
+            "numeric_statistics": {
+                "x": {
+                    "count": 20,
+                    "mean": 99.0,
+                    "std": 77.0,
+                    "min": 1.0,
+                    "max": 200.0,
+                    "median": 13.0,
+                    "skewness": 0.4,
+                    "distribution_fit": {"available": True, "best_fit": {"name": "norm"}},
+                }
+            },
+            "execution": {"scope": "bounded_deep_statistics"},
+        },
+    }
+
+    result = reuse_streaming_exact_statistics(payload)
+    stats = result["deep_statistics_v2"]["numeric_statistics"]["x"]
+
+    assert stats["count"] == 100
+    assert stats["mean"] == 12.5
+    assert stats["std"] == 3.25
+    assert stats["min"] == -2.0
+    assert stats["max"] == 30.0
+    assert stats["median"] == 13.0
+    assert stats["skewness"] == 0.4
+    assert stats["distribution_fit"]["best_fit"]["name"] == "norm"
+    assert stats["summary_provenance"]["backend"] == "rust"
+    assert set(stats["summary_provenance"]["reused_exact_fields"]) == {
+        "count",
+        "mean",
+        "std",
+        "min",
+        "max",
+    }
+    reuse = result["deep_statistics_v2"]["execution"]["exact_once_reuse"]
+    assert reuse["enabled"] is True
+    assert reuse["columns_reused"] == 1
+
+
+def test_noop_when_deep_statistics_are_not_applicable():
+    payload = {
+        "profile": {
+            "numeric_summary": {"x": {"count": 10, "mean": 2.0}},
+            "streaming_metadata": {"enabled": True, "numeric_backend": "rust"},
+        },
+        "deep_statistics_v2": None,
+    }
+
+    assert reuse_streaming_exact_statistics(payload) is payload

From fdb61343a1ad5b1ef6564528c333d79ee46ea809 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:47:16 +0530
Subject: [PATCH 396/439] bench: report native backend and exact-once reuse

---
 benchmarks/benchmark_real_wide_parquet.py | 27 ++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/benchmarks/benchmark_real_wide_parquet.py b/benchmarks/benchmark_real_wide_parquet.py
index 0797c6c..f0f8880 100644
--- a/benchmarks/benchmark_real_wide_parquet.py
+++ b/benchmarks/benchmark_real_wide_parquet.py
@@ -86,7 +86,10 @@ def generate_dataset(
             patterns = [_pattern_arrow(index, row_ids) for index in range(PATTERN_COUNT)]
             arrays = [patterns[index % PATTERN_COUNT] for index in range(columns)]
             batch = pa.RecordBatch.from_arrays(arrays, schema=schema)
-            writer.write_table(pa.Table.from_batches([batch], schema=schema), row_group_size=len(row_ids))
+            writer.write_table(
+                pa.Table.from_batches([batch], schema=schema),
+                row_group_size=len(row_ids),
+            )
     finally:
         writer.close()
 
@@ -150,9 +153,7 @@ def _accuracy_checks(result: dict[str, Any], *, rows: int) -> dict[str, Any]:
         if observed_mean is not None and truth["mean"] is not None:
             mean_errors.append(abs(float(observed_mean) - float(truth["mean"])))
 
-    expected_health_missing = (
-        expected_missing_total / max(rows * len(columns), 1) * 100.0
-    )
+    expected_health_missing = expected_missing_total / max(rows * len(columns), 1) * 100.0
     observed_health_missing = float(
         result.get("health", {}).get("details", {}).get("missing_percent", 0.0) or 0.0
     )
@@ -160,6 +161,7 @@ def _accuracy_checks(result: dict[str, Any], *, rows: int) -> dict[str, Any]:
     deep_stats = result.get("deep_statistics_v2", {})
     numeric_stats = deep_stats.get("numeric_statistics", {}) if isinstance(deep_stats, dict) else {}
     deep_mean_errors: list[float] = []
+    exact_once_columns = 0
     for column, summary in numeric_stats.items():
         if not isinstance(summary, dict) or summary.get("mean") is None:
             continue
@@ -167,6 +169,9 @@ def _accuracy_checks(result: dict[str, Any], *, rows: int) -> dict[str, Any]:
         truth_mean = expected[index % PATTERN_COUNT]["mean"]
         if truth_mean is not None:
             deep_mean_errors.append(abs(float(summary["mean"]) - float(truth_mean)))
+        provenance = summary.get("summary_provenance", {})
+        if isinstance(provenance, dict) and provenance.get("scope") == "full_stream":
+            exact_once_columns += 1
 
     timings = result.get("timings_ms", {})
     phase3 = timings.get("phase3_tasks", {}) if isinstance(timings, dict) else {}
@@ -181,6 +186,7 @@ def _accuracy_checks(result: dict[str, Any], *, rows: int) -> dict[str, Any]:
         "health_missing_percent": round(observed_health_missing, 6),
         "health_missing_abs_error": round(abs(observed_health_missing - expected_health_missing), 6),
         "deep_numeric_columns_checked": len(numeric_stats),
+        "deep_exact_once_columns": exact_once_columns,
         "deep_mean_max_abs_error": round(max(deep_mean_errors), 6) if deep_mean_errors else None,
         "deep_mean_mean_abs_error": round(float(np.mean(deep_mean_errors)), 6) if deep_mean_errors else None,
         "phase3_tasks_present": sorted(phase3) if isinstance(phase3, dict) else [],
@@ -212,8 +218,11 @@ def benchmark_dataset(path: Path, *, mode: str) -> dict[str, Any]:
     analysis_seconds = time.perf_counter() - started
 
     streaming = result.get("execution", {}).get("streaming", {})
+    profile = result.get("profile", {})
+    numeric_metadata = profile.get("numeric_summary_metadata", {})
+    streaming_metadata = profile.get("streaming_metadata", {})
     return {
-        "benchmark_schema_version": 1,
+        "benchmark_schema_version": 2,
         "workload": {
             "rows": rows,
             "columns": columns,
@@ -223,6 +232,14 @@ def benchmark_dataset(path: Path, *, mode: str) -> dict[str, Any]:
             "mode": mode,
             "source": "real_parquet_file",
         },
+        "backend": {
+            "numeric_backend": streaming_metadata.get("numeric_backend"),
+            "numeric_method": (
+                numeric_metadata.get("method")
+                if isinstance(numeric_metadata, dict)
+                else None
+            ),
+        },
         "analysis_seconds": round(analysis_seconds, 6),
         "pipeline_timings_ms": result.get("timings_ms", {}),
         "streaming_execution": streaming,

From 648a5d80bd8c3abe858c74c6f29ec8aab4e807cf Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:47:36 +0530
Subject: [PATCH 397/439] bench: compare native Rust and NumPy fallback on real
 wide data

---
 .../workflows/real-wide-parquet-benchmark.yml | 185 ++++++++++++------
 1 file changed, 127 insertions(+), 58 deletions(-)

diff --git a/.github/workflows/real-wide-parquet-benchmark.yml b/.github/workflows/real-wide-parquet-benchmark.yml
index 461f4de..8e1da24 100644
--- a/.github/workflows/real-wide-parquet-benchmark.yml
+++ b/.github/workflows/real-wide-parquet-benchmark.yml
@@ -7,6 +7,7 @@ on:
       - "benchmarks/benchmark_real_wide_parquet.py"
       - ".github/workflows/real-wide-parquet-benchmark.yml"
       - "src/framevitals/**"
+      - "rust/**"
   workflow_dispatch:
 
 permissions:
@@ -18,7 +19,7 @@ concurrency:
 
 jobs:
   benchmark:
-    name: Full pipeline 100k x 10k real Parquet
+    name: Native vs fallback full pipeline 100k x 10k real Parquet
     runs-on: ubuntu-latest
     timeout-minutes: 60
 
@@ -35,10 +36,34 @@ jobs:
           cache: pip
           cache-dependency-path: pyproject.toml
 
-      - name: Install benchmark runtime
+      - name: Configure Rust
+        run: |
+          rustup default stable
+          rustc --version
+          cargo --version
+
+      - name: Install benchmark and native build runtime
         run: |
           python -m pip install --upgrade pip
           pip install -e ".[arrow]"
+          pip install "maturin>=1.14,<2"
+
+      - name: Build and install FrameVitals native engine
+        run: |
+          maturin build \
+            --release \
+            --manifest-path rust/framevitals-py/Cargo.toml \
+            --interpreter python \
+            --out dist-native
+          pip install --force-reinstall --no-deps dist-native/*.whl
+          FRAMEVITALS_BACKEND=rust python - <<'PY'
+          from framevitals.backends import backend_status
+
+          status = backend_status()
+          assert status["native_available"] is True, status
+          assert status["selected"] == "rust", status
+          print(status)
+          PY
 
       - name: Generate real 100k x 10k Parquet dataset
         run: |
@@ -50,53 +75,86 @@ jobs:
             --row-group-rows 10000 \
             --output generation.json
 
-      - name: Run all four full analysis modes
+      - name: Run native Rust full analysis in all modes
+        env:
+          FRAMEVITALS_BACKEND: rust
+        run: |
+          for mode in quick standard deep research; do
+            echo "=== rust / ${mode} ==="
+            python benchmarks/benchmark_real_wide_parquet.py \
+              --dataset wide-100k-x-10k.parquet \
+              --mode "${mode}" \
+              --output "real-wide-rust-${mode}.json"
+          done
+
+      - name: Run NumPy fallback full analysis in all modes
         env:
           FRAMEVITALS_BACKEND: numpy
         run: |
           for mode in quick standard deep research; do
-            echo "=== ${mode} ==="
+            echo "=== numpy / ${mode} ==="
             python benchmarks/benchmark_real_wide_parquet.py \
               --dataset wide-100k-x-10k.parquet \
               --mode "${mode}" \
-              --output "real-wide-${mode}.json"
+              --output "real-wide-numpy-${mode}.json"
           done
 
-      - name: Validate full-pipeline accuracy
+      - name: Validate backend routing and full-pipeline accuracy
         run: |
           python - <<'PY'
           import json
           from pathlib import Path
 
           failures = []
-          for mode in ("quick", "standard", "deep", "research"):
-              payload = json.loads(Path(f"real-wide-{mode}.json").read_text())
-              accuracy = payload["accuracy"]
-              checked = accuracy["profiled_columns_checked"]
-              if accuracy["missing_count_exact_matches"] != checked:
-                  failures.append(f"{mode}: missing-count parity failed")
-              if accuracy["numeric_count_exact_matches"] != checked:
-                  failures.append(f"{mode}: numeric-count parity failed")
-              if accuracy["numeric_minmax_exact_matches"] != checked:
-                  failures.append(f"{mode}: min/max parity failed")
-              if (accuracy["profile_mean_max_abs_error"] or 0.0) > 0.01:
-                  failures.append(
-                      f"{mode}: profile mean max error {accuracy['profile_mean_max_abs_error']}"
-                  )
-              if accuracy["health_missing_abs_error"] > 0.02:
-                  failures.append(
-                      f"{mode}: health missingness error {accuracy['health_missing_abs_error']}"
+          for backend in ("rust", "numpy"):
+              for mode in ("quick", "standard", "deep", "research"):
+                  payload = json.loads(
+                      Path(f"real-wide-{backend}-{mode}.json").read_text()
                   )
-              if (accuracy["deep_mean_max_abs_error"] or 0.0) > 8.0:
-                  failures.append(
-                      f"{mode}: bounded deep-stat mean error {accuracy['deep_mean_max_abs_error']}"
-                  )
-              if payload["safety"]["full_materialization"] is not False:
-                  failures.append(f"{mode}: full materialization occurred")
+                  observed_backend = payload["backend"]["numeric_backend"]
+                  if observed_backend != backend:
+                      failures.append(
+                          f"{backend}/{mode}: routed to {observed_backend!r}"
+                      )
+
+                  accuracy = payload["accuracy"]
+                  checked = accuracy["profiled_columns_checked"]
+                  if accuracy["missing_count_exact_matches"] != checked:
+                      failures.append(f"{backend}/{mode}: missing-count parity failed")
+                  if accuracy["numeric_count_exact_matches"] != checked:
+                      failures.append(f"{backend}/{mode}: numeric-count parity failed")
+                  if accuracy["numeric_minmax_exact_matches"] != checked:
+                      failures.append(f"{backend}/{mode}: min/max parity failed")
+                  if (accuracy["profile_mean_max_abs_error"] or 0.0) > 0.01:
+                      failures.append(
+                          f"{backend}/{mode}: profile mean max error "
+                          f"{accuracy['profile_mean_max_abs_error']}"
+                      )
+                  if accuracy["health_missing_abs_error"] > 0.02:
+                      failures.append(
+                          f"{backend}/{mode}: health missingness error "
+                          f"{accuracy['health_missing_abs_error']}"
+                      )
+
+                  deep_checked = accuracy["deep_numeric_columns_checked"]
+                  if deep_checked:
+                      if accuracy["deep_exact_once_columns"] != deep_checked:
+                          failures.append(
+                              f"{backend}/{mode}: exact-once reuse missing for "
+                              f"{deep_checked - accuracy['deep_exact_once_columns']} columns"
+                          )
+                      if (accuracy["deep_mean_max_abs_error"] or 0.0) > 0.01:
+                          failures.append(
+                              f"{backend}/{mode}: deep exact mean error "
+                              f"{accuracy['deep_mean_max_abs_error']}"
+                          )
+
+                  if payload["safety"]["full_materialization"] is not False:
+                      failures.append(f"{backend}/{mode}: full materialization occurred")
 
           if failures:
               raise SystemExit("\n".join(failures))
-          print("All full-pipeline accuracy checks passed.")
+          print("Native and fallback full-pipeline accuracy checks passed.")
           PY
 
       - name: Publish benchmark summary
@@ -118,43 +176,54 @@ jobs:
               print(f"- Row groups: **{generation['row_groups']}**")
               print()
 
-          print("## Full analysis results")
+          print("## Native Rust vs NumPy fallback")
           print()
-          print("| Mode | Wall time | Profiled cols | Sample rows | Profile mean max error | Deep mean max error | Health missing error |")
-          print("|---|---:|---:|---:|---:|---:|---:|")
-          for mode in ("quick", "standard", "deep", "research"):
-              path = Path(f"real-wide-{mode}.json")
-              if not path.exists():
-                  print(f"| {mode.title()} | failed | - | - | - | - | - |")
-                  continue
-              payload = json.loads(path.read_text())
-              safety = payload["safety"]
-              accuracy = payload["accuracy"]
-              print(
-                  f"| {mode.title()} | {payload['analysis_seconds']:.3f}s | "
-                  f"{safety['profiled_columns']} | {safety['working_sample_rows']} | "
-                  f"{accuracy['profile_mean_max_abs_error']} | "
-                  f"{accuracy['deep_mean_max_abs_error']} | "
-                  f"{accuracy['health_missing_abs_error']} |"
-              )
-              print()
-              print(f"### {mode.title()} pipeline stages")
-              for name, milliseconds in sorted(payload.get("pipeline_timings_ms", {}).items()):
-                  if isinstance(milliseconds, (int, float)):
-                      print(f"- {name}: {milliseconds / 1000:.3f}s")
+          print("| Backend | Mode | Wall time | Profiled cols | Sample rows | Profile mean max error | Deep mean max error |")
+          print("|---|---|---:|---:|---:|---:|---:|")
+          for backend in ("rust", "numpy"):
+              for mode in ("quick", "standard", "deep", "research"):
+                  path = Path(f"real-wide-{backend}-{mode}.json")
+                  if not path.exists():
+                      print(f"| {backend} | {mode.title()} | failed | - | - | - | - |")
+                      continue
+                  payload = json.loads(path.read_text())
+                  safety = payload["safety"]
+                  accuracy = payload["accuracy"]
+                  print(
+                      f"| {backend} | {mode.title()} | {payload['analysis_seconds']:.3f}s | "
+                      f"{safety['profiled_columns']} | {safety['working_sample_rows']} | "
+                      f"{accuracy['profile_mean_max_abs_error']} | "
+                      f"{accuracy['deep_mean_max_abs_error']} |"
+                  )
               print()
+
+          for backend in ("rust", "numpy"):
+              for mode in ("quick", "standard", "deep", "research"):
+                  path = Path(f"real-wide-{backend}-{mode}.json")
+                  if not path.exists():
+                      continue
+                  payload = json.loads(path.read_text())
+                  print(f"### {backend} / {mode.title()} pipeline stages")
+                  for name, milliseconds in sorted(payload.get("pipeline_timings_ms", {}).items()):
+                      if isinstance(milliseconds, (int, float)):
+                          print(f"- {name}: {milliseconds / 1000:.3f}s")
+                  print()
           PY
 
       - name: Upload benchmark results
         if: always()
         uses: actions/upload-artifact@v6
         with:
-          name: framevitals-real-wide-100k-x-10k
+          name: framevitals-real-wide-100k-x-10k-native-vs-numpy
           path: |
             generation.json
-            real-wide-quick.json
-            real-wide-standard.json
-            real-wide-deep.json
-            real-wide-research.json
+            real-wide-rust-quick.json
+            real-wide-rust-standard.json
+            real-wide-rust-deep.json
+            real-wide-rust-research.json
+            real-wide-numpy-quick.json
+            real-wide-numpy-standard.json
+            real-wide-numpy-deep.json
+            real-wide-numpy-research.json
           if-no-files-found: ignore
           retention-days: 30

From 2fb5243fae1f8fc28a52e4dfdfbf13fc093181ea Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:49:55 +0530
Subject: [PATCH 398/439] test: use valid minimum quality sample size

---
 tests/test_projected_streaming_semantics.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/test_projected_streaming_semantics.py b/tests/test_projected_streaming_semantics.py
index 329b0e3..cf515f7 100644
--- a/tests/test_projected_streaming_semantics.py
+++ b/tests/test_projected_streaming_semantics.py
@@ -47,14 +47,16 @@ def test_ml_readiness_uses_profiled_width_for_projected_missingness():
 
 def test_streaming_quality_reports_only_columns_actually_checked():
     profile = _projected_profile(rows=200, profiled_columns=64)
-    sample = pd.DataFrame({f"c{i}": [i, i + 1, i + 2] for i in range(64)})
+    sample = pd.DataFrame(
+        {f"c{i}": [i + row for row in range(10)] for i in range(64)}
+    )
 
     result = run_streaming_quality_diagnostics(
         sample,
         profile=profile,
         source_rows=200,
         source_columns=10_000,
-        max_sample_rows=3,
+        max_sample_rows=10,
         max_columns=100,
     )
 

From 3f8bb39dd462245a975e99dcebf6fa18d998782c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:56:36 +0530
Subject: [PATCH 399/439] perf: profile Arrow record batches directly in Rust

---
 rust/framevitals-core/src/fused_profile.rs |  54 ++++++++-
 rust/framevitals-py/Cargo.toml             |   3 +-
 rust/framevitals-py/src/lib.rs             |  74 ++++++++++++-
 src/framevitals/backends.py                |  19 +++-
 src/framevitals/streaming_profile.py       | 121 ++++++++++++++++-----
 tests/test_native_arrow_batch_profile.py   |  34 ++++++
 6 files changed, 270 insertions(+), 35 deletions(-)
 create mode 100644 tests/test_native_arrow_batch_profile.py

diff --git a/rust/framevitals-core/src/fused_profile.rs b/rust/framevitals-core/src/fused_profile.rs
index 798b060..fac7411 100644
--- a/rust/framevitals-core/src/fused_profile.rs
+++ b/rust/framevitals-core/src/fused_profile.rs
@@ -7,8 +7,8 @@
 use std::collections::{BTreeMap, BTreeSet};
 
 use arrow_array::{
-    Array, Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, UInt32Array,
-    UInt64Array,
+    Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array,
+    RecordBatch, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
 };
 use arrow_schema::DataType;
 
@@ -25,7 +25,9 @@ impl NumericProfileState {
     pub fn observe(&mut self, value: Option<f64>, stream_id: u64, sequence: u64) {
         self.moments.observe(value);
         if let Some(value) = value {
-            self.sketches.observe(value, stream_id, sequence);
+            if value.is_finite() {
+                self.sketches.observe(value, stream_id, sequence);
+            }
         }
     }
 
@@ -110,6 +112,14 @@ pub fn profile_record_batch(batch: &RecordBatch, partition_id: u64) -> BatchProf
                 .as_any()
                 .downcast_ref::<Int32Array>()
                 .map(|values| scan_profile_primitive!(values, stream_id)),
+            DataType::Int16 => array
+                .as_any()
+                .downcast_ref::<Int16Array>()
+                .map(|values| scan_profile_primitive!(values, stream_id)),
+            DataType::Int8 => array
+                .as_any()
+                .downcast_ref::<Int8Array>()
+                .map(|values| scan_profile_primitive!(values, stream_id)),
             DataType::UInt64 => array
                 .as_any()
                 .downcast_ref::<UInt64Array>()
@@ -118,6 +128,14 @@ pub fn profile_record_batch(batch: &RecordBatch, partition_id: u64) -> BatchProf
                 .as_any()
                 .downcast_ref::<UInt32Array>()
                 .map(|values| scan_profile_primitive!(values, stream_id)),
+            DataType::UInt16 => array
+                .as_any()
+                .downcast_ref::<UInt16Array>()
+                .map(|values| scan_profile_primitive!(values, stream_id)),
+            DataType::UInt8 => array
+                .as_any()
+                .downcast_ref::<UInt8Array>()
+                .map(|values| scan_profile_primitive!(values, stream_id)),
             _ => None,
         };
 
@@ -138,7 +156,9 @@ pub fn profile_record_batch(batch: &RecordBatch, partition_id: u64) -> BatchProf
 mod tests {
     use std::sync::Arc;
 
-    use arrow_array::{ArrayRef, Float64Array, Int64Array, RecordBatch, StringArray};
+    use arrow_array::{
+        ArrayRef, Float64Array, Int16Array, Int64Array, RecordBatch, StringArray, UInt8Array,
+    };
 
     use super::profile_record_batch;
 
@@ -176,6 +196,32 @@ mod tests {
         assert!(state.skipped_columns.contains("label"));
     }
 
+    #[test]
+    fn fused_profile_supports_compact_integer_arrays() {
+        let batch = RecordBatch::try_from_iter(vec![
+            (
+                "small_signed",
+                Arc::new(Int16Array::from(vec![Some(-2), Some(4), None, Some(8)])) as ArrayRef,
+            ),
+            (
+                "small_unsigned",
+                Arc::new(UInt8Array::from(vec![Some(1), Some(2), Some(3), Some(4)])) as ArrayRef,
+            ),
+        ])
+        .expect("valid compact integer batch");
+
+        let state = profile_record_batch(&batch, 1);
+        let signed = state.profiles.get("small_signed").unwrap();
+        let unsigned = state.profiles.get("small_unsigned").unwrap();
+
+        assert_eq!(signed.moments.count, 3);
+        assert_eq!(signed.moments.missing, 1);
+        assert_eq!(signed.moments.minimum, Some(-2.0));
+        assert_eq!(signed.moments.maximum, Some(8.0));
+        assert_eq!(unsigned.moments.count, 4);
+        assert_eq!(unsigned.moments.mean, 2.5);
+    }
+
     #[test]
     fn fused_partition_merge_preserves_compact_profile_semantics() {
         let left = batch(
diff --git a/rust/framevitals-py/Cargo.toml b/rust/framevitals-py/Cargo.toml
index cec8855..ca137aa 100644
--- a/rust/framevitals-py/Cargo.toml
+++ b/rust/framevitals-py/Cargo.toml
@@ -12,5 +12,6 @@ path = "src/lib.rs"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-framevitals-core = { path = "../framevitals-core" }
+framevitals-core = { path = "../framevitals-core", features = ["arrow"] }
 pyo3 = { version = "0.29", features = ["abi3-py311"] }
+pyo3-arrow = "0.19"
diff --git a/rust/framevitals-py/src/lib.rs b/rust/framevitals-py/src/lib.rs
index 77401c6..a48833c 100644
--- a/rust/framevitals-py/src/lib.rs
+++ b/rust/framevitals-py/src/lib.rs
@@ -2,12 +2,14 @@
 
 mod string_accumulator;
 
+use framevitals_core::fused_profile::{profile_record_batch, BatchProfileState};
 use framevitals_core::sketches::NumericSketchState;
 use framevitals_core::NumericState;
 use pyo3::buffer::PyBuffer;
 use pyo3::exceptions::PyBufferError;
 use pyo3::prelude::*;
 use pyo3::types::PyDict;
+use pyo3_arrow::PyRecordBatch;
 use string_accumulator::StringAccumulator;
 
 fn exact_state_dict<'py>(py: Python<'py>, state: &NumericState) -> PyResult<Bound<'py, PyDict>> {
@@ -63,6 +65,30 @@ fn profile_dict<'py>(
     Ok(payload)
 }
 
+fn batch_profile_dict<'py>(
+    py: Python<'py>,
+    state: &BatchProfileState,
+) -> PyResult<Bound<'py, PyDict>> {
+    let payload = PyDict::new(py);
+    payload.set_item("backend", "rust")?;
+    payload.set_item("rows", state.rows)?;
+
+    let profiles = PyDict::new(py);
+    for (name, profile) in &state.profiles {
+        let observations = profile.moments.count + profile.moments.missing + profile.moments.infinite;
+        profiles.set_item(
+            name,
+            profile_dict(py, &profile.moments, &profile.sketches, observations)?,
+        )?;
+    }
+    payload.set_item("profiles", profiles)?;
+    payload.set_item(
+        "skipped_columns",
+        state.skipped_columns.iter().cloned().collect::<Vec<_>>(),
+    )?;
+    Ok(payload)
+}
+
 fn checked_buffer<'py>(values: &Bound<'py, PyAny>) -> PyResult<PyBuffer<f64>> {
     let buffer = PyBuffer::<f64>::get(values)?;
     if buffer.dimensions() != 1 {
@@ -96,7 +122,9 @@ fn update_states(
     for value in slice {
         let value = value.get();
         state.observe(Some(value));
-        sketches.observe(value, stream_id, *sequence);
+        if value.is_finite() {
+            sketches.observe(value, stream_id, *sequence);
+        }
         *sequence = sequence.wrapping_add(1);
     }
     Ok(buffer.item_count() as u64)
@@ -155,6 +183,49 @@ impl NumericAccumulator {
     }
 }
 
+/// Persistent zero-copy Arrow batch profiler.
+///
+/// ``pyo3-arrow`` imports ``pyarrow.RecordBatch`` through the Arrow PyCapsule
+/// interface. Numeric primitive arrays are scanned directly in Rust without
+/// constructing NumPy float64 copies or dispatching once per column in Python.
+#[pyclass]
+struct ArrowBatchProfileAccumulator {
+    state: BatchProfileState,
+    next_partition_id: u64,
+}
+
+#[pymethods]
+impl ArrowBatchProfileAccumulator {
+    #[new]
+    fn new() -> Self {
+        Self {
+            state: BatchProfileState::default(),
+            next_partition_id: 0,
+        }
+    }
+
+    fn update(&mut self, batch: PyRecordBatch) {
+        let partition = profile_record_batch(batch.as_ref(), self.next_partition_id);
+        let current = std::mem::take(&mut self.state);
+        self.state = current.merge(partition);
+        self.next_partition_id = self.next_partition_id.wrapping_add(1);
+    }
+
+    fn snapshot(&self, py: Python<'_>) -> PyResult<Py<PyDict>> {
+        Ok(batch_profile_dict(py, &self.state)?.unbind())
+    }
+
+    fn reset(&mut self) {
+        self.state = BatchProfileState::default();
+        self.next_partition_id = 0;
+    }
+
+    #[getter]
+    fn rows(&self) -> u64 {
+        self.state.rows
+    }
+}
+
 #[pyfunction]
 fn numeric_state_f64(py: Python<'_>, values: &Bound<'_, PyAny>) -> PyResult<Py<PyDict>> {
     let buffer = checked_buffer(values)?;
@@ -203,6 +274,7 @@ fn backend_info() -> (&'static str, &'static str) {
 #[pymodule]
 fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_class::<NumericAccumulator>()?;
+    module.add_class::<ArrowBatchProfileAccumulator>()?;
     module.add_class::<StringAccumulator>()?;
     module.add_function(wrap_pyfunction!(numeric_state_f64, module)?)?;
     module.add_function(wrap_pyfunction!(numeric_profile_f64, module)?)?;
diff --git a/src/framevitals/backends.py b/src/framevitals/backends.py
index e433fbd..951dbe7 100644
--- a/src/framevitals/backends.py
+++ b/src/framevitals/backends.py
@@ -1,8 +1,8 @@
 """Backend routing for FrameVitals native and NumPy kernels.
 
-The router keeps optional native/GPU dependencies behind lazy imports. Normal
-installations remain NumPy-first, while a compiled ``framevitals._native``
-extension can transparently accelerate compatible streaming scans.
+The router keeps optional native/GPU dependencies behind lazy imports. Compiled
+FrameVitals native kernels are preferred when available; NumPy remains the
+portable reference/fallback backend for compatible primitives.
 """
 
 from __future__ import annotations
@@ -139,6 +139,19 @@ def create_numeric_accumulator(*, stream_id: int = 0):
     return native.NumericAccumulator(stream_id=int(stream_id))
 
 
+def create_arrow_batch_profile_accumulator():
+    """Create the zero-copy native Arrow RecordBatch profiler when available.
+
+    Returning ``None`` for an older native extension preserves compatibility
+    with installations that only expose the per-column float64 accumulator.
+    """
+    if resolve_numeric_backend() != "rust":
+        return None
+    native = import_module("framevitals._native")
+    accumulator = getattr(native, "ArrowBatchProfileAccumulator", None)
+    return accumulator() if accumulator is not None else None
+
+
 def create_string_accumulator():
     """Create a persistent native UTF-8 sketch accumulator when available."""
     if resolve_numeric_backend() != "rust":
diff --git a/src/framevitals/streaming_profile.py b/src/framevitals/streaming_profile.py
index 4249f0a..f85c988 100644
--- a/src/framevitals/streaming_profile.py
+++ b/src/framevitals/streaming_profile.py
@@ -2,8 +2,9 @@
 
 The streaming path keeps full-row work to mergeable column state and retains
 only a bounded, evenly spaced row sample for analyses that still require row
-relationships (duplicate estimation and correlation). Native builds can also
-consume Arrow UTF-8 buffers directly for full-file categorical sketches.
+relationships (duplicate estimation and correlation). Native builds consume
+Arrow RecordBatches directly for fused numeric profiling and Arrow UTF-8 buffers
+for full-file categorical sketches when those kernels are available.
 """
 
 from __future__ import annotations
@@ -17,6 +18,7 @@
 
 from framevitals.analysis_state import NumericColumnState
 from framevitals.backends import (
+    create_arrow_batch_profile_accumulator,
     create_numeric_accumulator,
     create_string_accumulator,
     numeric_state,
@@ -106,6 +108,20 @@ def _string_columns(schema) -> list[str]:
     ]
 
 
+def _native_fused_numeric_schema_supported(schema) -> bool:
+    """Return whether every Arrow numeric field is supported by the fused Rust scan."""
+    pa = _require_pyarrow()
+    for field in schema:
+        dtype = field.type
+        if pa.types.is_integer(dtype):
+            if int(dtype.bit_width) not in {8, 16, 32, 64}:
+                return False
+        elif pa.types.is_floating(dtype):
+            if int(dtype.bit_width) not in {32, 64}:
+                return False
+    return True
+
+
 def numeric_columns_for_streaming_source(source: StreamingDatasetSource) -> list[str]:
     """Return Arrow numeric column names without reading row data."""
     schema_method = getattr(source, "schema", None)
@@ -116,7 +132,11 @@ def numeric_columns_for_streaming_source(source: StreamingDatasetSource) -> list
 
 
 def _arrow_numeric_to_float64(array) -> np.ndarray:
-    """Convert one bounded Arrow numeric array to a contiguous float64 buffer."""
+    """Convert one bounded Arrow numeric array to a contiguous float64 buffer.
+
+    This is a compatibility fallback for NumPy and older native extensions. New
+    native builds profile supported Arrow primitive arrays directly in Rust.
+    """
     try:
         values = array.to_numpy(zero_copy_only=False)
         converted = np.asarray(values, dtype=np.float64)
@@ -380,6 +400,11 @@ def build_streaming_profile(
     columns = [field.name for field in schema]
     numeric_cols, categorical_cols, date_cols = _column_groups(schema)
     numeric_col_set = set(numeric_cols)
+    non_numeric_columns = [
+        (index, column)
+        for index, column in enumerate(columns)
+        if column not in numeric_col_set
+    ]
     string_cols = _string_columns(schema)
     dtypes = {field.name: str(field.type) for field in schema}
 
@@ -404,6 +429,7 @@ def build_streaming_profile(
         and should_use_full_stream_numpy_sketch(int(rows), len(numeric_cols))
     )
 
+    native_batch_accumulator = None
     native_accumulators: dict[str, Any] = {}
     native_string_accumulators: dict[str, Any] = {}
     python_states = {column: NumericColumnState() for column in numeric_cols}
@@ -413,16 +439,20 @@ def build_streaming_profile(
         else {}
     )
     if selected_backend == "rust":
-        native_accumulators = {
-            column: create_numeric_accumulator(stream_id=index)
-            for index, column in enumerate(numeric_cols)
-        }
+        if numeric_cols and _native_fused_numeric_schema_supported(schema):
+            native_batch_accumulator = create_arrow_batch_profile_accumulator()
+        if native_batch_accumulator is None:
+            native_accumulators = {
+                column: create_numeric_accumulator(stream_id=index)
+                for index, column in enumerate(numeric_cols)
+            }
         native_string_accumulators = {
             column: accumulator
             for column in string_cols
             if (accumulator := create_string_accumulator()) is not None
         }
 
+    native_arrow_fused = native_batch_accumulator is not None
     offset = 0
     batches_scanned = 0
     for batch in source.iter_batches(batch_size=effective_batch_size):
@@ -434,27 +464,36 @@ def build_streaming_profile(
         if sampled is not None and not sampled.empty:
             sample_frames.append(sampled)
 
-        for index, column in enumerate(columns):
-            array = batch.column(index)
-            if column not in numeric_col_set:
+        if native_arrow_fused:
+            native_batch_accumulator.update(batch)
+            for index, column in non_numeric_columns:
+                array = batch.column(index)
                 missing_counts[column] += int(array.null_count)
                 string_accumulator = native_string_accumulators.get(column)
                 if string_accumulator is not None:
                     _update_native_string_accumulator(string_accumulator, array)
-                continue
-
-            values = _arrow_numeric_to_float64(array)
-            accumulator = native_accumulators.get(column)
-            if accumulator is not None:
-                accumulator.update_f64(values)
-            else:
-                numeric_payload = numeric_state(values, backend="numpy")
-                python_states[column] = python_states[column].merge(
-                    _state_from_payload(numeric_payload)
-                )
-                quantile_sketch = numpy_quantile_sketches.get(column)
-                if quantile_sketch is not None:
-                    quantile_sketch.update(values)
+        else:
+            for index, column in enumerate(columns):
+                array = batch.column(index)
+                if column not in numeric_col_set:
+                    missing_counts[column] += int(array.null_count)
+                    string_accumulator = native_string_accumulators.get(column)
+                    if string_accumulator is not None:
+                        _update_native_string_accumulator(string_accumulator, array)
+                    continue
+
+                values = _arrow_numeric_to_float64(array)
+                accumulator = native_accumulators.get(column)
+                if accumulator is not None:
+                    accumulator.update_f64(values)
+                else:
+                    numeric_payload = numeric_state(values, backend="numpy")
+                    python_states[column] = python_states[column].merge(
+                        _state_from_payload(numeric_payload)
+                    )
+                    quantile_sketch = numpy_quantile_sketches.get(column)
+                    if quantile_sketch is not None:
+                        quantile_sketch.update(values)
 
         offset += int(batch.num_rows)
 
@@ -474,8 +513,32 @@ def build_streaming_profile(
     numeric_summary: dict[str, dict[str, Any]] = {}
     quantile_accuracy = None
     if selected_backend == "rust":
+        if native_arrow_fused:
+            batch_payload = dict(native_batch_accumulator.snapshot())
+            if int(batch_payload.get("rows", -1)) != int(rows):
+                raise ValueError(
+                    "Native Arrow profiler row count did not match streaming source metadata."
+                )
+            native_profiles = {
+                str(column): dict(profile)
+                for column, profile in dict(batch_payload.get("profiles", {})).items()
+            }
+            missing_numeric = [
+                column for column in numeric_cols if column not in native_profiles
+            ]
+            if missing_numeric:
+                raise RuntimeError(
+                    "Native Arrow profiler skipped supported numeric columns: "
+                    + ", ".join(missing_numeric[:5])
+                )
+        else:
+            native_profiles = {
+                column: dict(native_accumulators[column].snapshot())
+                for column in numeric_cols
+            }
+
         for column in numeric_cols:
-            numeric_payload = dict(native_accumulators[column].snapshot())
+            numeric_payload = native_profiles[column]
             numeric_summary[column] = _summary_from_native_snapshot(numeric_payload)
             missing_counts[column] = int(numeric_payload["missing"])
             quantile_accuracy = numeric_payload.get("quantiles", {}).get(
@@ -484,7 +547,12 @@ def build_streaming_profile(
             )
         numeric_metadata = {
             "backend": "rust",
-            "method": "native_streaming_accumulator",
+            "method": (
+                "native_arrow_fused_record_batch"
+                if native_arrow_fused
+                else "native_streaming_accumulator"
+            ),
+            "zero_copy_arrow_input": bool(native_arrow_fused),
             "approximate_quantiles": True,
             "quantile_relative_accuracy": quantile_accuracy,
             "quantile_source": "full_stream_sketch",
@@ -642,6 +710,7 @@ def build_streaming_profile(
             ),
             "sample_strategy": "evenly_spaced_global_rows",
             "numeric_backend": selected_backend,
+            "native_arrow_fused_numeric": bool(native_arrow_fused),
             "numpy_full_stream_quantile_sketches": bool(use_numpy_quantile_sketches),
             "numpy_numeric_sketch_cell_budget": int(PYTHON_NUMERIC_SKETCH_CELL_BUDGET),
             "native_string_sketch_columns": native_categorical_columns,
diff --git a/tests/test_native_arrow_batch_profile.py b/tests/test_native_arrow_batch_profile.py
new file mode 100644
index 0000000..b61cee8
--- /dev/null
+++ b/tests/test_native_arrow_batch_profile.py
@@ -0,0 +1,34 @@
+import pytest
+
+pa = pytest.importorskip("pyarrow")
+native = pytest.importorskip("framevitals._native")
+
+
+def test_native_arrow_batch_accumulator_profiles_int16_without_numpy_bridge():
+    accumulator_type = getattr(native, "ArrowBatchProfileAccumulator", None)
+    assert accumulator_type is not None
+
+    batch = pa.record_batch(
+        [
+            pa.array([1, 2, None, 4], type=pa.int16()),
+            pa.array([10.0, 20.0, 30.0, 40.0], type=pa.float64()),
+            pa.array(["a", "b", "c", "d"]),
+        ],
+        names=["small", "wide", "label"],
+    )
+    accumulator = accumulator_type()
+    accumulator.update(batch)
+    payload = dict(accumulator.snapshot())
+    profiles = {
+        str(name): dict(value)
+        for name, value in dict(payload["profiles"]).items()
+    }
+
+    assert payload["rows"] == 4
+    assert profiles["small"]["count"] == 3
+    assert profiles["small"]["missing"] == 1
+    assert profiles["small"]["mean"] == pytest.approx(7 / 3)
+    assert profiles["small"]["minimum"] == 1.0
+    assert profiles["small"]["maximum"] == 4.0
+    assert profiles["wide"]["mean"] == pytest.approx(25.0)
+    assert "label" in payload["skipped_columns"]

From b30eca407690873d7765dce2132f04be1664d743 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 09:57:58 +0530
Subject: [PATCH 400/439] style: format native Arrow bridge

---
 rust/framevitals-core/src/fused_profile.rs | 4 ++--
 rust/framevitals-py/src/lib.rs             | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/rust/framevitals-core/src/fused_profile.rs b/rust/framevitals-core/src/fused_profile.rs
index fac7411..28ddf68 100644
--- a/rust/framevitals-core/src/fused_profile.rs
+++ b/rust/framevitals-core/src/fused_profile.rs
@@ -7,8 +7,8 @@
 use std::collections::{BTreeMap, BTreeSet};
 
 use arrow_array::{
-    Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array,
-    RecordBatch, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+    Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, RecordBatch,
+    UInt16Array, UInt32Array, UInt64Array, UInt8Array,
 };
 use arrow_schema::DataType;
 
diff --git a/rust/framevitals-py/src/lib.rs b/rust/framevitals-py/src/lib.rs
index a48833c..867ecda 100644
--- a/rust/framevitals-py/src/lib.rs
+++ b/rust/framevitals-py/src/lib.rs
@@ -75,7 +75,8 @@ fn batch_profile_dict<'py>(
 
     let profiles = PyDict::new(py);
     for (name, profile) in &state.profiles {
-        let observations = profile.moments.count + profile.moments.missing + profile.moments.infinite;
+        let observations =
+            profile.moments.count + profile.moments.missing + profile.moments.infinite;
         profiles.set_item(
             name,
             profile_dict(py, &profile.moments, &profile.sketches, observations)?,

From 2930bc6dcc076fd891f79943d18c227c986f7473 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 10:04:29 +0530
Subject: [PATCH 401/439] perf: specialize native streaming profile sketches

---
 rust/framevitals-core/src/fused_profile.rs | 66 ++++++++++------------
 rust/framevitals-py/src/lib.rs             | 33 ++++++++++-
 tests/test_native_arrow_batch_profile.py   |  5 ++
 3 files changed, 67 insertions(+), 37 deletions(-)

diff --git a/rust/framevitals-core/src/fused_profile.rs b/rust/framevitals-core/src/fused_profile.rs
index 28ddf68..dad7c03 100644
--- a/rust/framevitals-core/src/fused_profile.rs
+++ b/rust/framevitals-core/src/fused_profile.rs
@@ -1,8 +1,10 @@
 //! Fused Arrow numeric profiling.
 //!
 //! One pass over each supported numeric Arrow column updates exact moments and
-//! bounded sketches together. This avoids a separate full-column scan for every
-//! statistic and produces states that can be merged across batches/partitions.
+//! the one sketch consumed by the streaming profile: mergeable log quantiles.
+//! Richer HLL/heavy-hitter/reservoir sketches remain available through the
+//! standalone numeric profiling APIs, but the full-stream dataframe profiler
+//! avoids paying for statistics it never reads.
 
 use std::collections::{BTreeMap, BTreeSet};
 
@@ -12,21 +14,21 @@ use arrow_array::{
 };
 use arrow_schema::DataType;
 
-use crate::sketches::NumericSketchState;
+use crate::sketches::LogQuantileSketch;
 use crate::NumericState;
 
 #[derive(Debug, Clone, PartialEq, Default)]
 pub struct NumericProfileState {
     pub moments: NumericState,
-    pub sketches: NumericSketchState,
+    pub quantiles: LogQuantileSketch,
 }
 
 impl NumericProfileState {
-    pub fn observe(&mut self, value: Option<f64>, stream_id: u64, sequence: u64) {
+    pub fn observe(&mut self, value: Option<f64>) {
         self.moments.observe(value);
         if let Some(value) = value {
             if value.is_finite() {
-                self.sketches.observe(value, stream_id, sequence);
+                self.quantiles.observe(value);
             }
         }
     }
@@ -35,7 +37,7 @@ impl NumericProfileState {
     pub fn merge(self, other: Self) -> Self {
         Self {
             moments: self.moments.merge(other.moments),
-            sketches: self.sketches.merge(other.sketches),
+            quantiles: self.quantiles.merge(other.quantiles),
         }
     }
 }
@@ -63,14 +65,14 @@ impl BatchProfileState {
 }
 
 macro_rules! scan_profile_primitive {
-    ($array:expr, $stream_id:expr) => {{
+    ($array:expr) => {{
         let array = $array;
         let mut profile = NumericProfileState::default();
         for index in 0..array.len() {
             if array.is_null(index) {
-                profile.observe(None, $stream_id, index as u64);
+                profile.observe(None);
             } else {
-                profile.observe(Some(array.value(index) as f64), $stream_id, index as u64);
+                profile.observe(Some(array.value(index) as f64));
             }
         }
         profile
@@ -79,10 +81,10 @@ macro_rules! scan_profile_primitive {
 
 /// Scan supported primitive numeric columns in one fused pass.
 ///
-/// `partition_id` must be stable and distinct for independently scanned
-/// partitions when deterministic reservoirs are expected to merge without
-/// priority collisions. Moments and the other sketches do not depend on it.
-pub fn profile_record_batch(batch: &RecordBatch, partition_id: u64) -> BatchProfileState {
+/// ``partition_id`` is retained in the public signature for compatibility with
+/// earlier fused-profile callers. Moments and log quantiles are deterministic
+/// without partition-specific state.
+pub fn profile_record_batch(batch: &RecordBatch, _partition_id: u64) -> BatchProfileState {
     let mut output = BatchProfileState {
         rows: batch.num_rows() as u64,
         ..BatchProfileState::default()
@@ -91,51 +93,48 @@ pub fn profile_record_batch(batch: &RecordBatch, partition_id: u64) -> BatchProf
     for (index, field) in batch.schema().fields().iter().enumerate() {
         let name = field.name().clone();
         let array = batch.column(index);
-        let stream_id = partition_id
-            .wrapping_mul(0x9E37_79B9_7F4A_7C15)
-            .wrapping_add(index as u64);
 
         let profile = match field.data_type() {
             DataType::Float64 => array
                 .as_any()
                 .downcast_ref::<Float64Array>()
-                .map(|values| scan_profile_primitive!(values, stream_id)),
+                .map(|values| scan_profile_primitive!(values)),
             DataType::Float32 => array
                 .as_any()
                 .downcast_ref::<Float32Array>()
-                .map(|values| scan_profile_primitive!(values, stream_id)),
+                .map(|values| scan_profile_primitive!(values)),
             DataType::Int64 => array
                 .as_any()
                 .downcast_ref::<Int64Array>()
-                .map(|values| scan_profile_primitive!(values, stream_id)),
+                .map(|values| scan_profile_primitive!(values)),
             DataType::Int32 => array
                 .as_any()
                 .downcast_ref::<Int32Array>()
-                .map(|values| scan_profile_primitive!(values, stream_id)),
+                .map(|values| scan_profile_primitive!(values)),
             DataType::Int16 => array
                 .as_any()
                 .downcast_ref::<Int16Array>()
-                .map(|values| scan_profile_primitive!(values, stream_id)),
+                .map(|values| scan_profile_primitive!(values)),
             DataType::Int8 => array
                 .as_any()
                 .downcast_ref::<Int8Array>()
-                .map(|values| scan_profile_primitive!(values, stream_id)),
+                .map(|values| scan_profile_primitive!(values)),
             DataType::UInt64 => array
                 .as_any()
                 .downcast_ref::<UInt64Array>()
-                .map(|values| scan_profile_primitive!(values, stream_id)),
+                .map(|values| scan_profile_primitive!(values)),
             DataType::UInt32 => array
                 .as_any()
                 .downcast_ref::<UInt32Array>()
-                .map(|values| scan_profile_primitive!(values, stream_id)),
+                .map(|values| scan_profile_primitive!(values)),
             DataType::UInt16 => array
                 .as_any()
                 .downcast_ref::<UInt16Array>()
-                .map(|values| scan_profile_primitive!(values, stream_id)),
+                .map(|values| scan_profile_primitive!(values)),
             DataType::UInt8 => array
                 .as_any()
                 .downcast_ref::<UInt8Array>()
-                .map(|values| scan_profile_primitive!(values, stream_id)),
+                .map(|values| scan_profile_primitive!(values)),
             _ => None,
         };
 
@@ -177,7 +176,7 @@ mod tests {
     }
 
     #[test]
-    fn fused_profile_updates_moments_and_sketches_together() {
+    fn fused_profile_updates_moments_and_quantiles_together() {
         let batch = batch(
             vec![Some(1.0), Some(2.0), None, Some(2.0), Some(f64::INFINITY)],
             vec![Some(1), Some(2), Some(3), Some(4), Some(5)],
@@ -190,9 +189,8 @@ mod tests {
         assert_eq!(value.moments.missing, 1);
         assert_eq!(value.moments.infinite, 1);
         assert!((value.moments.mean - 5.0 / 3.0).abs() < 1e-12);
-        assert!(value.sketches.cardinality.estimate() > 1.5);
-        assert_eq!(value.sketches.quantiles.count(), 3);
-        assert!(!value.sketches.reservoir.is_empty());
+        assert_eq!(value.quantiles.count(), 3);
+        assert!(value.quantiles.quantile(0.5).is_some());
         assert!(state.skipped_columns.contains("label"));
     }
 
@@ -223,7 +221,7 @@ mod tests {
     }
 
     #[test]
-    fn fused_partition_merge_preserves_compact_profile_semantics() {
+    fn fused_partition_merge_preserves_profile_semantics() {
         let left = batch(
             (0..1_000).map(|value| Some(value as f64)).collect(),
             (0..1_000).map(|value| Some(value as i64)).collect(),
@@ -239,9 +237,7 @@ mod tests {
         assert_eq!(merged.rows, 2_000);
         assert_eq!(value.moments.count, 2_000);
         assert!((value.moments.mean - 999.5).abs() < 1e-12);
-        assert!(value.sketches.cardinality.estimate() > 1_800.0);
-        let median = value.sketches.quantiles.quantile(0.5).unwrap();
+        let median = value.quantiles.quantile(0.5).unwrap();
         assert!(median > 900.0 && median < 1_100.0);
-        assert_eq!(value.sketches.reservoir.len(), 256);
     }
 }
diff --git a/rust/framevitals-py/src/lib.rs b/rust/framevitals-py/src/lib.rs
index 867ecda..70db753 100644
--- a/rust/framevitals-py/src/lib.rs
+++ b/rust/framevitals-py/src/lib.rs
@@ -3,7 +3,7 @@
 mod string_accumulator;
 
 use framevitals_core::fused_profile::{profile_record_batch, BatchProfileState};
-use framevitals_core::sketches::NumericSketchState;
+use framevitals_core::sketches::{LogQuantileSketch, NumericSketchState};
 use framevitals_core::NumericState;
 use pyo3::buffer::PyBuffer;
 use pyo3::exceptions::PyBufferError;
@@ -65,6 +65,34 @@ fn profile_dict<'py>(
     Ok(payload)
 }
 
+fn streaming_profile_dict<'py>(
+    py: Python<'py>,
+    state: &NumericState,
+    quantile_sketch: &LogQuantileSketch,
+    observations: u64,
+) -> PyResult<Bound<'py, PyDict>> {
+    let payload = exact_state_dict(py, state)?;
+    payload.set_item("backend", "rust")?;
+    payload.set_item("observations", observations)?;
+    payload.set_item("sketch_policy", "moments_and_log_quantiles")?;
+
+    let quantiles = PyDict::new(py);
+    for (name, q) in [
+        ("p01", 0.01),
+        ("p05", 0.05),
+        ("p25", 0.25),
+        ("p50", 0.50),
+        ("p75", 0.75),
+        ("p95", 0.95),
+        ("p99", 0.99),
+    ] {
+        quantiles.set_item(name, quantile_sketch.quantile(q))?;
+    }
+    quantiles.set_item("relative_accuracy", quantile_sketch.relative_accuracy())?;
+    payload.set_item("quantiles", quantiles)?;
+    Ok(payload)
+}
+
 fn batch_profile_dict<'py>(
     py: Python<'py>,
     state: &BatchProfileState,
@@ -72,6 +100,7 @@ fn batch_profile_dict<'py>(
     let payload = PyDict::new(py);
     payload.set_item("backend", "rust")?;
     payload.set_item("rows", state.rows)?;
+    payload.set_item("sketch_policy", "moments_and_log_quantiles")?;
 
     let profiles = PyDict::new(py);
     for (name, profile) in &state.profiles {
@@ -79,7 +108,7 @@ fn batch_profile_dict<'py>(
             profile.moments.count + profile.moments.missing + profile.moments.infinite;
         profiles.set_item(
             name,
-            profile_dict(py, &profile.moments, &profile.sketches, observations)?,
+            streaming_profile_dict(py, &profile.moments, &profile.quantiles, observations)?,
         )?;
     }
     payload.set_item("profiles", profiles)?;
diff --git a/tests/test_native_arrow_batch_profile.py b/tests/test_native_arrow_batch_profile.py
index b61cee8..e3e4fb2 100644
--- a/tests/test_native_arrow_batch_profile.py
+++ b/tests/test_native_arrow_batch_profile.py
@@ -25,10 +25,15 @@ def test_native_arrow_batch_accumulator_profiles_int16_without_numpy_bridge():
     }
 
     assert payload["rows"] == 4
+    assert payload["sketch_policy"] == "moments_and_log_quantiles"
     assert profiles["small"]["count"] == 3
     assert profiles["small"]["missing"] == 1
     assert profiles["small"]["mean"] == pytest.approx(7 / 3)
     assert profiles["small"]["minimum"] == 1.0
     assert profiles["small"]["maximum"] == 4.0
+    assert profiles["small"]["sketch_policy"] == "moments_and_log_quantiles"
+    assert "heavy_hitters" not in profiles["small"]
+    assert "reservoir" not in profiles["small"]
+    assert profiles["small"]["quantiles"]["p50"] is not None
     assert profiles["wide"]["mean"] == pytest.approx(25.0)
     assert "label" in payload["skipped_columns"]

From 861cca1999b094d3afa12a74a2eb70b4dd5f8b66 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 10:13:32 +0530
Subject: [PATCH 402/439] perf: skip duplicate materialized core work for
 streams

---
 src/framevitals/streaming_bounded_pipeline.py | 301 ++++++++++++++++++
 src/framevitals/streaming_pipeline.py         |  60 ++--
 tests/test_streaming_bounded_pipeline.py      |  81 +++++
 3 files changed, 421 insertions(+), 21 deletions(-)
 create mode 100644 src/framevitals/streaming_bounded_pipeline.py
 create mode 100644 tests/test_streaming_bounded_pipeline.py

diff --git a/src/framevitals/streaming_bounded_pipeline.py b/src/framevitals/streaming_bounded_pipeline.py
new file mode 100644
index 0000000..eb6a517
--- /dev/null
+++ b/src/framevitals/streaming_bounded_pipeline.py
@@ -0,0 +1,301 @@
+"""Bounded module scheduler for streaming analyses.
+
+Streaming sources already own profile, role, health, readiness, quality, and
+source-shape state. Re-entering the materialized pipeline on the retained sample
+would recompute those facts and then throw them away. This scheduler runs only
+modules that genuinely require row-level values while consuming the source-level
+execution budget chosen by the streaming planner.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, Callable
+
+import pandas as pd
+
+from framevitals.advanced_indicators import calculate_advanced_indicators
+from framevitals.budgeted_analysis import (
+    run_budgeted_anomalies,
+    run_budgeted_deep_statistics,
+    run_budgeted_time_series,
+)
+from framevitals.config import VALID_MODULES
+from framevitals.execution import ExecutionBudget, derive_execution_budget
+from framevitals.model_leaderboard import run_model_leaderboard
+from framevitals.pipeline import _result_status, _safe_call, _skipped_module
+from framevitals.target_intelligence import run_target_intelligence
+from framevitals.text_profile import profile_text_columns
+
+
+def run_streaming_bounded_modules(
+    dataframe: pd.DataFrame,
+    *,
+    dataset_id: str,
+    original_filename: str,
+    analysis_mode: str,
+    target_column: str | None,
+    parallel_workers: int,
+    source_budget: ExecutionBudget,
+    column_roles: dict[str, Any],
+    skip_ai: bool,
+    disabled_modules: set[str] | tuple[str, ...] | list[str] | None = None,
+) -> dict[str, Any]:
+    """Run only row-dependent modules on the bounded streaming sample.
+
+    Statistical adapters receive ``source_budget`` so ultra-wide source limits
+    remain authoritative even though the retained sample is much narrower. The
+    scheduler's concurrency limit, however, is derived from the bounded sample
+    because that is the memory footprint actually resident during parallel work.
+    """
+    overall_start = time.perf_counter()
+    timings_ms: dict[str, Any] = {}
+    disabled = set(disabled_modules or ())
+    unknown = sorted(disabled - VALID_MODULES)
+    if unknown:
+        raise ValueError("Unknown disabled module(s): " + ", ".join(unknown))
+    if dataframe.empty:
+        raise ValueError("Bounded streaming sample is empty.")
+    if target_column is not None and target_column not in dataframe.columns:
+        raise ValueError(f"Target column not found: {target_column}")
+
+    module_status: dict[str, str] = {
+        name: "pending" for name in sorted(VALID_MODULES)
+    }
+
+    def module_enabled(name: str) -> bool:
+        return name not in disabled
+
+    t0 = time.perf_counter()
+    advanced = calculate_advanced_indicators(dataframe)
+    timings_ms["advanced"] = (time.perf_counter() - t0) * 1000
+
+    phase3_modules: list[tuple[str, str, Callable[[], Any]]] = [
+        (
+            "deep_statistics",
+            "deep_statistics_v2",
+            lambda: run_budgeted_deep_statistics(dataframe, budget=source_budget),
+        ),
+        (
+            "anomaly_detection",
+            "anomalies_v2",
+            lambda: run_budgeted_anomalies(dataframe, budget=source_budget),
+        ),
+        (
+            "time_series",
+            "time_series",
+            lambda: run_budgeted_time_series(
+                dataframe,
+                budget=source_budget,
+                target_column=target_column,
+            ),
+        ),
+        ("text_profile", "text_profile", lambda: profile_text_columns(dataframe)),
+    ]
+
+    phase3_results: dict[str, Any] = {}
+    bounded_parallel_budget = derive_execution_budget(
+        len(dataframe),
+        len(dataframe.columns),
+        mode=analysis_mode,
+    )
+    phase3_worker_limit = max(
+        1,
+        min(
+            int(parallel_workers),
+            int(bounded_parallel_budget.max_memory_heavy_parallelism),
+        ),
+    )
+
+    if analysis_mode in {"standard", "deep", "research"}:
+        tasks: list[tuple[str, str, Callable[[], Any]]] = []
+        for module, result_key, fn in phase3_modules:
+            if module_enabled(module):
+                tasks.append((module, result_key, fn))
+                module_status[module] = "scheduled"
+            else:
+                phase3_results[result_key] = _skipped_module(module)
+                module_status[module] = "disabled"
+
+        per_task_ms: dict[str, float] = {}
+        if tasks:
+            phase3_start = time.perf_counter()
+            with ThreadPoolExecutor(max_workers=phase3_worker_limit) as executor:
+                futures = {
+                    executor.submit(_safe_call, result_key, fn): (module, result_key)
+                    for module, result_key, fn in tasks
+                }
+                for future in as_completed(futures):
+                    module, result_key = futures[future]
+                    name, value, elapsed = future.result()
+                    phase3_results[result_key] = value
+                    per_task_ms[name] = elapsed
+                    module_status[module] = _result_status(value)
+            timings_ms["phase3_parallel_total"] = (
+                time.perf_counter() - phase3_start
+            ) * 1000
+        else:
+            timings_ms["phase3_parallel_total"] = 0.0
+        timings_ms["phase3_tasks"] = per_task_ms
+    else:
+        timings_ms["phase3_parallel_total"] = 0.0
+        timings_ms["phase3_tasks"] = {}
+        for module, _, _ in phase3_modules:
+            module_status[module] = "not_applicable"
+
+    deep_statistics_v2 = phase3_results.get("deep_statistics_v2")
+    anomalies_v2 = phase3_results.get("anomalies_v2")
+    time_series_analysis = phase3_results.get("time_series")
+    text_profile = phase3_results.get("text_profile")
+
+    target_intelligence = None
+    model_leaderboard = None
+    explainability = None
+
+    if target_column:
+        if module_enabled("target_intelligence"):
+            t0 = time.perf_counter()
+            _, target_intelligence, _ = _safe_call(
+                "target_intelligence",
+                lambda: run_target_intelligence(
+                    dataframe,
+                    target_column=target_column,
+                    column_roles=column_roles,
+                ),
+            )
+            timings_ms["target_intelligence"] = (time.perf_counter() - t0) * 1000
+            module_status["target_intelligence"] = _result_status(target_intelligence)
+        else:
+            target_intelligence = _skipped_module("target_intelligence")
+            timings_ms["target_intelligence"] = 0.0
+            module_status["target_intelligence"] = "disabled"
+    else:
+        module_status["target_intelligence"] = "not_applicable"
+
+    modeling_applicable = bool(
+        target_column and analysis_mode in {"standard", "deep", "research"}
+    )
+    if modeling_applicable:
+        if module_enabled("modeling"):
+            t0 = time.perf_counter()
+            _, model_leaderboard, _ = _safe_call(
+                "model_leaderboard",
+                lambda: run_model_leaderboard(dataframe, target_column=target_column),
+            )
+            timings_ms["model_leaderboard"] = (time.perf_counter() - t0) * 1000
+            module_status["modeling"] = _result_status(model_leaderboard)
+        else:
+            model_leaderboard = _skipped_module("modeling")
+            timings_ms["model_leaderboard"] = 0.0
+            module_status["modeling"] = "disabled"
+    else:
+        module_status["modeling"] = "not_applicable"
+
+    winner_available = (
+        isinstance(model_leaderboard, dict)
+        and model_leaderboard.get("available")
+        and model_leaderboard.get("winner")
+    )
+    if winner_available:
+        if module_enabled("explainability"):
+            t0 = time.perf_counter()
+
+            def run_explainability():
+                from framevitals.explainability import explain_winner
+
+                return explain_winner(
+                    dataframe,
+                    target_column=target_column,
+                    leaderboard_result=model_leaderboard,
+                    dataset_id=dataset_id,
+                )
+
+            _, explainability, _ = _safe_call("explainability", run_explainability)
+            timings_ms["explainability"] = (time.perf_counter() - t0) * 1000
+            module_status["explainability"] = _result_status(explainability)
+        else:
+            explainability = _skipped_module("explainability")
+            timings_ms["explainability"] = 0.0
+            module_status["explainability"] = "disabled"
+    elif not module_enabled("explainability"):
+        explainability = _skipped_module("explainability")
+        module_status["explainability"] = "disabled"
+    else:
+        module_status["explainability"] = "not_applicable"
+
+    # Full AI generation consumes the final full-stream profile/signals and is
+    # therefore intentionally left to the outer streaming orchestrator. The
+    # public analyze API currently requests skip_ai=True on this path.
+    if not module_enabled("ai"):
+        ai_report = {
+            "source": "disabled",
+            "text": "AI report disabled by configuration.",
+            "deferred": False,
+        }
+        module_status["ai"] = "disabled"
+    elif skip_ai:
+        ai_report = {
+            "source": "skipped",
+            "text": "AI report skipped.",
+            "deferred": False,
+        }
+        module_status["ai"] = "skipped_by_caller"
+    else:
+        ai_env = os.environ.get(
+            "FRAMEVITALS_ANALYZE_AI",
+            os.environ.get("DATALENS_ANALYZE_AI", "0"),
+        )
+        enabled = ai_env.strip().lower() in {"1", "true", "yes"}
+        ai_report = {
+            "source": "deferred",
+            "text": "",
+            "deferred": True,
+            "reason": (
+                "Streaming AI interpretation is deferred until full-stream core "
+                "signals are assembled."
+                if enabled
+                else "AI analysis is not enabled by environment."
+            ),
+        }
+        module_status["ai"] = "deferred"
+    timings_ms["ai_report"] = 0.0
+
+    # Core streaming modules are intentionally not executed here. Their status
+    # is filled by the outer streaming pipeline after reuse.
+    for module in ("quality_diagnostics", "cleaning", "charts"):
+        if module in disabled:
+            module_status[module] = "disabled"
+
+    timings_ms["total"] = (time.perf_counter() - overall_start) * 1000
+    return {
+        "dataset_id": dataset_id,
+        "filename": original_filename,
+        "analysis_mode": analysis_mode,
+        "artifacts_enabled": False,
+        "execution": {
+            "disabled_modules": sorted(disabled),
+            "module_status": module_status,
+            "budget": source_budget.to_dict(),
+            "phase3_worker_limit": int(phase3_worker_limit),
+            "bounded_scheduler": {
+                "enabled": True,
+                "core_reprofiled": False,
+                "source_budget_scope": "source_shape",
+                "parallelism_budget_scope": "bounded_sample",
+                "sample_rows": int(len(dataframe)),
+                "sample_columns": int(len(dataframe.columns)),
+            },
+        },
+        "advanced": advanced,
+        "deep_statistics_v2": deep_statistics_v2,
+        "anomalies_v2": anomalies_v2,
+        "target_intelligence": target_intelligence,
+        "model_leaderboard": model_leaderboard,
+        "explainability": explainability,
+        "time_series": time_series_analysis,
+        "text_profile": text_profile,
+        "ai_report": ai_report,
+        "timings_ms": timings_ms,
+    }
diff --git a/src/framevitals/streaming_pipeline.py b/src/framevitals/streaming_pipeline.py
index b5f62d1..f49a27b 100644
--- a/src/framevitals/streaming_pipeline.py
+++ b/src/framevitals/streaming_pipeline.py
@@ -1,11 +1,11 @@
 """Full FrameVitals orchestration for streaming dataset sources.
 
 The streaming pipeline performs one full-source Arrow scan to build reusable
-profile state, retains one bounded deterministic working sample, and runs legacy
-row-dependent modules only on that sample. Full-source facts are overlaid back
-onto the stable analysis result shape, and every bounded module is explicitly
-scoped so downstream consumers never mistake sample-derived diagnostics for
-full-data execution.
+profile state, retains one bounded deterministic working sample, and runs only
+row-dependent modules on that sample. Full-source facts are reused instead of
+re-entering the materialized pandas pipeline, and every bounded module is
+explicitly scoped so downstream consumers never mistake sample-derived
+diagnostics for full-data execution.
 """
 
 from __future__ import annotations
@@ -24,9 +24,9 @@
 )
 from framevitals.health_score import calculate_health_score_from_profile_sample
 from framevitals.ml_readiness import calculate_ml_readiness_from_profile
-from framevitals.pipeline import run_full_analysis
 from framevitals.signal_engine import build_signals
 from framevitals.sources import DatasetMetadata, StreamingDatasetSource
+from framevitals.streaming_bounded_pipeline import run_streaming_bounded_modules
 from framevitals.streaming_profile import build_streaming_profile
 from framevitals.streaming_quality import run_streaming_quality_diagnostics
 from framevitals.streaming_roles import infer_streaming_column_roles
@@ -257,13 +257,14 @@ def run_streaming_analysis(
         else source
     )
 
-    profile_start = time.perf_counter()
+    core_timings: dict[str, float] = {}
+    t0 = time.perf_counter()
     profile, working_sample = build_streaming_profile(
         profile_source,
         sample_rows=working_rows,
         return_sample=True,
     )
-    streaming_profile_ms = (time.perf_counter() - profile_start) * 1000
+    core_timings["streaming_profile"] = (time.perf_counter() - t0) * 1000
 
     profiled_columns = int(len(working_sample.columns))
     column_sampled = profiled_columns < columns
@@ -286,11 +287,21 @@ def run_streaming_analysis(
     if target_column is not None and target_column not in working_sample.columns:
         raise ValueError(f"Target column not found: {target_column}")
 
+    t0 = time.perf_counter()
     role_payload = infer_streaming_column_roles(working_sample, profile=profile)
     column_roles = role_payload["columns"]
     roles_summary = summarize_roles(column_roles)
+    core_timings["column_roles"] = (time.perf_counter() - t0) * 1000
+
+    t0 = time.perf_counter()
     health = calculate_health_score_from_profile_sample(profile, working_sample)
+    core_timings["health"] = (time.perf_counter() - t0) * 1000
+
+    t0 = time.perf_counter()
     ml_readiness = calculate_ml_readiness_from_profile(profile)
+    core_timings["ml_readiness"] = (time.perf_counter() - t0) * 1000
+
+    t0 = time.perf_counter()
     quality_diagnostics = (
         run_streaming_quality_diagnostics(
             working_sample,
@@ -307,13 +318,18 @@ def run_streaming_analysis(
             "reason": "Disabled by configuration.",
         }
     )
+    core_timings["quality_diagnostics"] = (time.perf_counter() - t0) * 1000
 
+    t0 = time.perf_counter()
     dataset_signals = detect_dataset_signals(
         working_sample,
         profile,
         column_roles=column_roles,
         source_shape=(rows, columns),
     )
+    core_timings["dataset_signals"] = (time.perf_counter() - t0) * 1000
+
+    t0 = time.perf_counter()
     analysis_selection = select_analyses(
         signals=dataset_signals,
         analysis_mode=analysis_mode,
@@ -333,21 +349,22 @@ def run_streaming_analysis(
         "profiled_columns": profiled_columns,
         "column_sampled": column_sampled,
     }
+    core_timings["analysis_selection"] = (time.perf_counter() - t0) * 1000
 
-    # Cleaning/charts are internally suppressed on the bounded sample. Charts
-    # are irrelevant here because this path is only selected when artifacts are
-    # disabled; cleaning would otherwise describe a transformed sample as though
-    # it represented the complete source.
+    # Cleaning/charts are internally suppressed on the bounded sample. The
+    # streaming scheduler runs only modules that genuinely require row values;
+    # profile/roles/health/readiness/quality above are never recomputed.
     internal_disabled = set(user_disabled) | {"cleaning", "charts"}
-    sample_payload = run_full_analysis(
+    sample_payload = run_streaming_bounded_modules(
+        working_sample,
         dataset_id=dataset_id,
         original_filename=original_filename,
         analysis_mode=analysis_mode,
-        skip_ai=skip_ai,
         target_column=target_column,
         parallel_workers=parallel_workers,
-        dataframe=working_sample,
-        write_artifacts=False,
+        source_budget=budget,
+        column_roles=column_roles,
+        skip_ai=skip_ai,
         disabled_modules=tuple(sorted(internal_disabled)),
     )
 
@@ -362,7 +379,9 @@ def run_streaming_analysis(
         )
 
     advanced = sample_payload.get("advanced")
+    t0 = time.perf_counter()
     signals = build_signals(profile, health, ml_readiness, advanced)
+    core_timings["signals"] = (time.perf_counter() - t0) * 1000
 
     cleaning = _streaming_cleaning_result(
         enabled_by_user="cleaning" not in user_disabled,
@@ -378,10 +397,9 @@ def run_streaming_analysis(
     module_status["charts"] = (
         "disabled" if "charts" in user_disabled else "not_applicable"
     )
-    if "quality_diagnostics" in user_disabled:
-        module_status["quality_diagnostics"] = "disabled"
-    else:
-        module_status["quality_diagnostics"] = "ran"
+    module_status["quality_diagnostics"] = (
+        "disabled" if "quality_diagnostics" in user_disabled else "ran"
+    )
 
     module_scope: dict[str, str] = {}
     for module in (
@@ -456,7 +474,7 @@ def run_streaming_analysis(
     })
 
     timings = dict(sample_payload.get("timings_ms", {}))
-    timings["streaming_profile"] = streaming_profile_ms
+    timings.update(core_timings)
     timings["total"] = (time.perf_counter() - overall_start) * 1000
 
     sample_payload.update({
diff --git a/tests/test_streaming_bounded_pipeline.py b/tests/test_streaming_bounded_pipeline.py
new file mode 100644
index 0000000..04c0514
--- /dev/null
+++ b/tests/test_streaming_bounded_pipeline.py
@@ -0,0 +1,81 @@
+import pandas as pd
+import pytest
+
+pa = pytest.importorskip("pyarrow")
+pq = pytest.importorskip("pyarrow.parquet")
+
+import framevitals
+import framevitals.analysis_api as analysis_api
+import framevitals.pipeline as materialized_pipeline
+
+
+def test_streaming_analysis_does_not_reenter_materialized_core(tmp_path, monkeypatch):
+    path = tmp_path / "bounded-only.parquet"
+    frame = pd.DataFrame({
+        "x": range(2_000),
+        "y": [value * 2 for value in range(2_000)],
+        "group": [f"g-{value % 5}" for value in range(2_000)],
+    })
+    pq.write_table(pa.Table.from_pandas(frame, preserve_index=False), path, row_group_size=333)
+
+    def fail(*args, **kwargs):
+        raise AssertionError("streaming analysis re-entered materialized core work")
+
+    # A streaming file must not fall back to the materialized public dispatcher.
+    monkeypatch.setattr(analysis_api, "run_full_analysis", fail)
+    # Nor may the bounded scheduler recompute core pandas profile/role/health/quality state.
+    monkeypatch.setattr(materialized_pipeline, "build_profile", fail)
+    monkeypatch.setattr(materialized_pipeline, "infer_column_roles", fail)
+    monkeypatch.setattr(materialized_pipeline, "calculate_health_score", fail)
+    monkeypatch.setattr(materialized_pipeline, "calculate_ml_readiness", fail)
+    monkeypatch.setattr(materialized_pipeline, "run_quality_diagnostics", fail)
+
+    result = framevitals.analyze(path, mode="quick", artifacts=False, workers=2)
+
+    scheduler = result["execution"]["bounded_scheduler"]
+    assert scheduler["enabled"] is True
+    assert scheduler["core_reprofiled"] is False
+    assert scheduler["source_budget_scope"] == "source_shape"
+    assert scheduler["parallelism_budget_scope"] == "bounded_sample"
+    assert result["profile"]["shape"] == {"rows": 2_000, "columns": 3}
+    assert result["execution"]["streaming"]["full_materialization"] is False
+    assert "profile" not in result["timings_ms"]
+    assert result["timings_ms"]["streaming_profile"] > 0
+
+
+def test_streaming_bounded_scheduler_uses_source_budget_for_ultra_wide_limits(tmp_path):
+    # The focused unit check is shape-only: the source budget must retain the
+    # ultra-wide relationship cap even though the retained sample is narrower.
+    from framevitals.execution import derive_execution_budget
+    from framevitals.streaming_bounded_pipeline import run_streaming_bounded_modules
+
+    sample = pd.DataFrame({f"c{i}": range(100) for i in range(32)})
+    source_budget = derive_execution_budget(100_000, 10_000, mode="standard")
+    payload = run_streaming_bounded_modules(
+        sample,
+        dataset_id="test",
+        original_filename="<test>",
+        analysis_mode="standard",
+        target_column=None,
+        parallel_workers=4,
+        source_budget=source_budget,
+        column_roles={},
+        skip_ai=True,
+        disabled_modules={
+            "deep_statistics",
+            "anomaly_detection",
+            "time_series",
+            "text_profile",
+            "modeling",
+            "explainability",
+            "cleaning",
+            "charts",
+            "quality_diagnostics",
+        },
+    )
+
+    budget = payload["execution"]["budget"]
+    assert budget["rows"] == 100_000
+    assert budget["columns"] == 10_000
+    assert budget["relationship_pair_budget"] == 10
+    assert payload["execution"]["bounded_scheduler"]["sample_columns"] == 32

From c3ea3a76bf8698fdcf34b88169e41286115c4a57 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 10:20:57 +0530
Subject: [PATCH 403/439] perf: accelerate native log-quantile bin updates

---
 rust/framevitals-core/Cargo.toml      |  1 +
 rust/framevitals-core/src/sketches.rs | 81 +++++++++++++++++++++------
 2 files changed, 66 insertions(+), 16 deletions(-)

diff --git a/rust/framevitals-core/Cargo.toml b/rust/framevitals-core/Cargo.toml
index 332fe64..caebf74 100644
--- a/rust/framevitals-core/Cargo.toml
+++ b/rust/framevitals-core/Cargo.toml
@@ -17,3 +17,4 @@ arrow = ["dep:arrow-array", "dep:arrow-schema"]
 [dependencies]
 arrow-array = { version = "59.1", optional = true }
 arrow-schema = { version = "59.1", optional = true }
+rustc-hash = "2.1"
diff --git a/rust/framevitals-core/src/sketches.rs b/rust/framevitals-core/src/sketches.rs
index 53e7a51..05dc45d 100644
--- a/rust/framevitals-core/src/sketches.rs
+++ b/rust/framevitals-core/src/sketches.rs
@@ -7,6 +7,8 @@
 use std::cmp::Ordering;
 use std::collections::BTreeMap;
 
+use rustc_hash::FxHashMap;
+
 const DEFAULT_ZERO_THRESHOLD: f64 = 1.0e-12;
 
 #[inline]
@@ -111,9 +113,10 @@ impl Default for HyperLogLog {
 pub struct LogQuantileSketch {
     relative_accuracy: f64,
     log_gamma: f64,
+    inv_log_gamma: f64,
     zero_threshold: f64,
-    negative: BTreeMap<i32, u64>,
-    positive: BTreeMap<i32, u64>,
+    negative: FxHashMap<i32, u64>,
+    positive: FxHashMap<i32, u64>,
     zero_count: u64,
     count: u64,
 }
@@ -125,25 +128,30 @@ impl LogQuantileSketch {
             "relative accuracy must be between 0 and 1"
         );
         let gamma = (1.0 + relative_accuracy) / (1.0 - relative_accuracy);
+        let log_gamma = gamma.ln();
         Self {
             relative_accuracy,
-            log_gamma: gamma.ln(),
+            log_gamma,
+            inv_log_gamma: 1.0 / log_gamma,
             zero_threshold: DEFAULT_ZERO_THRESHOLD,
-            negative: BTreeMap::new(),
-            positive: BTreeMap::new(),
+            negative: FxHashMap::default(),
+            positive: FxHashMap::default(),
             zero_count: 0,
             count: 0,
         }
     }
 
+    #[inline]
     fn key(&self, magnitude: f64) -> i32 {
-        (magnitude.ln() / self.log_gamma).floor() as i32
+        (magnitude.ln() * self.inv_log_gamma).floor() as i32
     }
 
+    #[inline]
     fn representative(&self, key: i32) -> f64 {
         ((key as f64 + 0.5) * self.log_gamma).exp()
     }
 
+    #[inline]
     pub fn observe(&mut self, value: f64) {
         if !value.is_finite() {
             return;
@@ -175,14 +183,35 @@ impl LogQuantileSketch {
         self
     }
 
-    pub fn quantile(&self, q: f64) -> Option<f64> {
+    fn sorted_bins(&self) -> (Vec<(i32, u64)>, Vec<(i32, u64)>) {
+        let mut negative: Vec<_> = self
+            .negative
+            .iter()
+            .map(|(key, count)| (*key, *count))
+            .collect();
+        let mut positive: Vec<_> = self
+            .positive
+            .iter()
+            .map(|(key, count)| (*key, *count))
+            .collect();
+        negative.sort_unstable_by_key(|(key, _)| *key);
+        positive.sort_unstable_by_key(|(key, _)| *key);
+        (negative, positive)
+    }
+
+    fn quantile_from_sorted(
+        &self,
+        q: f64,
+        negative: &[(i32, u64)],
+        positive: &[(i32, u64)],
+    ) -> Option<f64> {
         if self.count == 0 || !(0.0..=1.0).contains(&q) {
             return None;
         }
         let target = (q * (self.count - 1) as f64).floor() as u64;
         let mut seen = 0_u64;
 
-        for (key, count) in self.negative.iter().rev() {
+        for (key, count) in negative.iter().rev() {
             if target < seen + count {
                 return Some(-self.representative(*key));
             }
@@ -192,23 +221,31 @@ impl LogQuantileSketch {
             return Some(0.0);
         }
         seen += self.zero_count;
-        for (key, count) in &self.positive {
+        for (key, count) in positive {
             if target < seen + count {
                 return Some(self.representative(*key));
             }
             seen += count;
         }
-        self.positive
-            .last_key_value()
+        positive
+            .last()
             .map(|(key, _)| self.representative(*key))
-            .or_else(|| {
-                self.negative
-                    .first_key_value()
-                    .map(|(key, _)| -self.representative(*key))
-            })
+            .or_else(|| negative.first().map(|(key, _)| -self.representative(*key)))
             .or(Some(0.0))
     }
 
+    pub fn quantile(&self, q: f64) -> Option<f64> {
+        let (negative, positive) = self.sorted_bins();
+        self.quantile_from_sorted(q, &negative, &positive)
+    }
+
+    pub fn quantiles(&self, qs: &[f64]) -> Vec<Option<f64>> {
+        let (negative, positive) = self.sorted_bins();
+        qs.iter()
+            .map(|q| self.quantile_from_sorted(*q, &negative, &positive))
+            .collect()
+    }
+
     pub fn count(&self) -> u64 {
         self.count
     }
@@ -464,6 +501,18 @@ mod tests {
         assert!(merged.bin_count() < 1_000);
     }
 
+    #[test]
+    fn batched_log_quantiles_match_individual_queries() {
+        let mut sketch = LogQuantileSketch::new(0.01);
+        for value in -5_000..=5_000 {
+            sketch.observe(value as f64);
+        }
+        let levels = [0.01, 0.25, 0.5, 0.75, 0.99];
+        let batched = sketch.quantiles(&levels);
+        let individual: Vec<_> = levels.iter().map(|q| sketch.quantile(*q)).collect();
+        assert_eq!(batched, individual);
+    }
+
     #[test]
     fn heavy_hitters_retain_dominant_candidates_after_merge() {
         let mut left = HeavyHittersSketch::new(8);

From bd685fa9a9f9a8a69bdf2bd32758b17b289803fa Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 10:24:07 +0530
Subject: [PATCH 404/439] style: simplify quantile bin type

---
 rust/framevitals-core/src/sketches.rs | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/rust/framevitals-core/src/sketches.rs b/rust/framevitals-core/src/sketches.rs
index 05dc45d..f6be1cb 100644
--- a/rust/framevitals-core/src/sketches.rs
+++ b/rust/framevitals-core/src/sketches.rs
@@ -10,6 +10,7 @@ use std::collections::BTreeMap;
 use rustc_hash::FxHashMap;
 
 const DEFAULT_ZERO_THRESHOLD: f64 = 1.0e-12;
+type QuantileBins = Vec<(i32, u64)>;
 
 #[inline]
 pub(crate) fn mix64(mut value: u64) -> u64 {
@@ -36,10 +37,7 @@ pub struct HyperLogLog {
 
 impl HyperLogLog {
     pub fn new(precision: u8) -> Self {
-        assert!(
-            (4..=16).contains(&precision),
-            "HLL precision must be 4..=16"
-        );
+        assert!((4..=16).contains(&precision), "HLL precision must be 4..=16");
         Self {
             precision,
             registers: vec![0; 1usize << precision],
@@ -183,13 +181,13 @@ impl LogQuantileSketch {
         self
     }
 
-    fn sorted_bins(&self) -> (Vec<(i32, u64)>, Vec<(i32, u64)>) {
-        let mut negative: Vec<_> = self
+    fn sorted_bins(&self) -> (QuantileBins, QuantileBins) {
+        let mut negative: QuantileBins = self
             .negative
             .iter()
             .map(|(key, count)| (*key, *count))
             .collect();
-        let mut positive: Vec<_> = self
+        let mut positive: QuantileBins = self
             .positive
             .iter()
             .map(|(key, count)| (*key, *count))
@@ -313,10 +311,7 @@ impl HeavyHittersSketch {
 
     #[must_use]
     pub fn merge(mut self, other: Self) -> Self {
-        assert_eq!(
-            self.capacity, other.capacity,
-            "heavy-hitter capacity mismatch"
-        );
+        assert_eq!(self.capacity, other.capacity, "heavy-hitter capacity mismatch");
         for (key, count) in other.counters {
             self.observe_weighted(key, count);
         }

From bd56c24722b2391af9b9c6d4e28d946930049c21 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 10:26:08 +0530
Subject: [PATCH 405/439] style: apply rustfmt to quantile sketch

---
 rust/framevitals-core/src/sketches.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/rust/framevitals-core/src/sketches.rs b/rust/framevitals-core/src/sketches.rs
index f6be1cb..967d382 100644
--- a/rust/framevitals-core/src/sketches.rs
+++ b/rust/framevitals-core/src/sketches.rs
@@ -37,7 +37,10 @@ pub struct HyperLogLog {
 
 impl HyperLogLog {
     pub fn new(precision: u8) -> Self {
-        assert!((4..=16).contains(&precision), "HLL precision must be 4..=16");
+        assert!(
+            (4..=16).contains(&precision),
+            "HLL precision must be 4..=16"
+        );
         Self {
             precision,
             registers: vec![0; 1usize << precision],
@@ -311,7 +314,10 @@ impl HeavyHittersSketch {
 
     #[must_use]
     pub fn merge(mut self, other: Self) -> Self {
-        assert_eq!(self.capacity, other.capacity, "heavy-hitter capacity mismatch");
+        assert_eq!(
+            self.capacity, other.capacity,
+            "heavy-hitter capacity mismatch"
+        );
         for (key, count) in other.counters {
             self.observe_weighted(key, count);
         }

From 61092f09ef51538917a6e7c7ffb22e6740a70cbb Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 10:32:46 +0530
Subject: [PATCH 406/439] perf: cache Parquet source metadata and handle

---
 src/framevitals/sources.py | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/src/framevitals/sources.py b/src/framevitals/sources.py
index 0bad2b3..c9bdffa 100644
--- a/src/framevitals/sources.py
+++ b/src/framevitals/sources.py
@@ -320,11 +320,23 @@ def load(self) -> pd.DataFrame:
 
 @dataclass(slots=True)
 class ParquetSource:
-    """Projection-aware, streaming Parquet source backed by optional PyArrow."""
+    """Projection-aware, streaming Parquet source backed by optional PyArrow.
+
+    A single source object owns one ``ParquetFile`` instance plus cached schema
+    and metadata. Ultra-wide files can have expensive footer/schema parsing, so
+    repeatedly reopening the same file inside ``inspect()``, ``schema()`` and
+    ``iter_batches()`` wastes work without improving correctness.
+    """
 
     path: Path
+    _parquet_file_cache: Any = field(default=None, init=False, repr=False)
+    _metadata_cache: DatasetMetadata | None = field(default=None, init=False, repr=False)
+    _schema_cache: Any = field(default=None, init=False, repr=False)
 
     def _parquet_file(self):
+        if self._parquet_file_cache is not None:
+            return self._parquet_file_cache
+
         _validate_file_path(self.path)
         try:
             import pyarrow.parquet as pq
@@ -333,12 +345,16 @@ def _parquet_file(self):
                 "Parquet streaming requires the optional Arrow capability. "
                 'Install it with: pip install "framevitals[arrow]"'
             ) from exc
-        return pq.ParquetFile(self.path)
+        self._parquet_file_cache = pq.ParquetFile(self.path)
+        return self._parquet_file_cache
 
     def inspect(self) -> DatasetMetadata:
+        if self._metadata_cache is not None:
+            return self._metadata_cache
+
         parquet_file = self._parquet_file()
         metadata = parquet_file.metadata
-        return DatasetMetadata(
+        self._metadata_cache = DatasetMetadata(
             name=self.path.name,
             kind="file",
             format="parquet",
@@ -349,10 +365,13 @@ def inspect(self) -> DatasetMetadata:
             supports_projection=True,
             supports_streaming=True,
         )
+        return self._metadata_cache
 
     def schema(self):
-        """Return the Arrow schema without reading all row data."""
-        return self._parquet_file().schema_arrow
+        """Return the cached Arrow schema without reading row data."""
+        if self._schema_cache is None:
+            self._schema_cache = self._parquet_file().schema_arrow
+        return self._schema_cache
 
     def iter_batches(
         self,
@@ -362,16 +381,14 @@ def iter_batches(
     ) -> Iterator[Any]:
         if batch_size < 1:
             raise ValueError("batch_size must be at least 1.")
-        parquet_file = self._parquet_file()
-        yield from parquet_file.iter_batches(
+        yield from self._parquet_file().iter_batches(
             batch_size=int(batch_size),
             columns=list(columns) if columns is not None else None,
             use_threads=True,
         )
 
     def load(self) -> pd.DataFrame:
-        parquet_file = self._parquet_file()
-        table = parquet_file.read(use_threads=True)
+        table = self._parquet_file().read(use_threads=True)
         dataframe = table.to_pandas()
         if dataframe.empty:
             raise ValueError(f"Dataset is empty: {self.path}")

From ccb95b7bceb5697511ddbd525e452a53c449def5 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 10:33:00 +0530
Subject: [PATCH 407/439] test: ensure Parquet metadata is parsed once per
 source

---
 tests/test_parquet_source_cache.py | 43 ++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 tests/test_parquet_source_cache.py

diff --git a/tests/test_parquet_source_cache.py b/tests/test_parquet_source_cache.py
new file mode 100644
index 0000000..64c7fdb
--- /dev/null
+++ b/tests/test_parquet_source_cache.py
@@ -0,0 +1,43 @@
+import pandas as pd
+import pytest
+
+pa = pytest.importorskip("pyarrow")
+pq = pytest.importorskip("pyarrow.parquet")
+
+from framevitals.sources import ParquetSource
+
+
+def test_parquet_source_reuses_one_parquet_file_handle(tmp_path, monkeypatch):
+    path = tmp_path / "cache.parquet"
+    frame = pd.DataFrame({
+        "x": range(100),
+        "y": [value * 2 for value in range(100)],
+    })
+    pq.write_table(pa.Table.from_pandas(frame, preserve_index=False), path, row_group_size=17)
+
+    real_parquet_file = pq.ParquetFile
+    calls = {"count": 0}
+
+    def counted_parquet_file(*args, **kwargs):
+        calls["count"] += 1
+        return real_parquet_file(*args, **kwargs)
+
+    monkeypatch.setattr(pq, "ParquetFile", counted_parquet_file)
+
+    source = ParquetSource(path)
+    first_metadata = source.inspect()
+    second_metadata = source.inspect()
+    first_schema = source.schema()
+    second_schema = source.schema()
+    first_batches = list(source.iter_batches(batch_size=23, columns=["x"]))
+    second_batches = list(source.iter_batches(batch_size=31, columns=["y"]))
+
+    assert calls["count"] == 1
+    assert first_metadata is second_metadata
+    assert first_schema is second_schema
+    assert first_metadata.rows == 100
+    assert first_metadata.columns == 2
+    assert sum(batch.num_rows for batch in first_batches) == 100
+    assert sum(batch.num_rows for batch in second_batches) == 100
+    assert first_batches[0].schema.names == ["x"]
+    assert second_batches[0].schema.names == ["y"]

From 0305250039d44c565fdde638bea94bb7d82d0344 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 10:41:34 +0530
Subject: [PATCH 408/439] fix: modernize Anderson-Darling normality diagnostics

---
 src/framevitals/deep_statistics_v2.py | 60 ++++++++++++++++++--
 tests/test_anderson_compatibility.py  | 81 +++++++++++++++++++++++++++
 2 files changed, 135 insertions(+), 6 deletions(-)
 create mode 100644 tests/test_anderson_compatibility.py

diff --git a/src/framevitals/deep_statistics_v2.py b/src/framevitals/deep_statistics_v2.py
index 9c3c446..0c9576b 100644
--- a/src/framevitals/deep_statistics_v2.py
+++ b/src/framevitals/deep_statistics_v2.py
@@ -114,6 +114,58 @@ def _outlier_flags(series: pd.Series) -> dict:
     return {"iqr": iqr_count, "z3": z3_count, "mad_z": mad_count, "n": n}
 
 
+def _legacy_anderson_critical_5pct(result: Any) -> float | int | None:
+    """Extract the legacy 5% critical value without assuming a fixed table index."""
+    critical_values = getattr(result, "critical_values", None)
+    significance_levels = getattr(result, "significance_level", None)
+    if critical_values is None:
+        return None
+
+    if significance_levels is not None:
+        try:
+            levels = np.asarray(significance_levels, dtype=float)
+            values = np.asarray(critical_values, dtype=float)
+            if levels.size and values.size == levels.size:
+                index = int(np.argmin(np.abs(levels - 5.0)))
+                return _safe_float(values[index])
+        except (TypeError, ValueError):
+            pass
+
+    try:
+        return _safe_float(critical_values[2])
+    except (IndexError, TypeError):
+        return None
+
+
+def _anderson_normality(series: pd.Series) -> dict[str, Any]:
+    """Run Anderson-Darling using the modern p-value API when available.
+
+    SciPy 1.17 introduced ``method`` and deprecated the legacy critical-value
+    result attributes. FrameVitals supports older SciPy releases too, so this
+    helper opts into the modern interpolated p-value and falls back only when
+    the installed SciPy does not recognize the keyword.
+    """
+    try:
+        result = stats.anderson(series, dist="norm", method="interpolate")
+    except TypeError as exc:
+        if "method" not in str(exc):
+            raise
+        result = stats.anderson(series, dist="norm")
+        return {
+            "statistic": _safe_float(result.statistic),
+            "p_value": None,
+            "critical_5pct": _legacy_anderson_critical_5pct(result),
+            "method": "legacy_critical_values",
+        }
+
+    return {
+        "statistic": _safe_float(result.statistic),
+        "p_value": _safe_float(result.pvalue),
+        "critical_5pct": None,
+        "method": "interpolate",
+    }
+
+
 def _normality(series: pd.Series) -> dict:
     """Run multiple normality tests, prefer Shapiro for small n, D'Agostino for medium."""
     s = series.dropna()
@@ -138,13 +190,9 @@ def _normality(series: pd.Series) -> dict:
         except Exception as exc:
             out["dagostino"] = {"error": str(exc)}
 
-    # Anderson-Darling
+    # Anderson-Darling. Explicit p-value method avoids SciPy >=1.17 deprecation warnings.
     try:
-        result = stats.anderson(s, dist="norm")
-        out["anderson"] = {
-            "statistic": _safe_float(result.statistic),
-            "critical_5pct": _safe_float(result.critical_values[2]),
-        }
+        out["anderson"] = _anderson_normality(s)
     except Exception as exc:
         out["anderson"] = {"error": str(exc)}
 
diff --git a/tests/test_anderson_compatibility.py b/tests/test_anderson_compatibility.py
new file mode 100644
index 0000000..ac239aa
--- /dev/null
+++ b/tests/test_anderson_compatibility.py
@@ -0,0 +1,81 @@
+from types import SimpleNamespace
+import warnings
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from framevitals import deep_statistics_v2
+from framevitals.deep_statistics_v2 import _anderson_normality, _normality
+
+
+def test_anderson_modern_api_returns_pvalue_without_futurewarning():
+    rng = np.random.default_rng(42)
+    sample = pd.Series(rng.normal(size=256))
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", FutureWarning)
+        result = _normality(sample)
+
+    anderson = result["anderson"]
+    assert anderson["method"] == "interpolate"
+    assert anderson["statistic"] is not None
+    assert anderson["critical_5pct"] is None
+    assert 0.0 <= anderson["p_value"] <= 1.0
+
+
+def test_anderson_legacy_api_falls_back_to_critical_values(monkeypatch):
+    calls = []
+
+    def fake_anderson(values, dist="norm", **kwargs):
+        calls.append(kwargs)
+        if "method" in kwargs:
+            raise TypeError("anderson() got an unexpected keyword argument 'method'")
+        return SimpleNamespace(
+            statistic=0.42,
+            critical_values=np.array([0.5, 0.6, 0.7, 0.8, 0.9]),
+            significance_level=np.array([15.0, 10.0, 5.0, 2.5, 1.0]),
+        )
+
+    monkeypatch.setattr(deep_statistics_v2.stats, "anderson", fake_anderson)
+    result = _anderson_normality(pd.Series(np.arange(20, dtype=float)))
+
+    assert calls == [{"method": "interpolate"}, {}]
+    assert result == {
+        "statistic": 0.42,
+        "p_value": None,
+        "critical_5pct": 0.7,
+        "method": "legacy_critical_values",
+    }
+
+
+def test_normality_verdict_still_prefers_shapiro(monkeypatch):
+    sample = pd.Series(np.linspace(-2.0, 2.0, 100))
+
+    monkeypatch.setattr(
+        deep_statistics_v2.stats,
+        "shapiro",
+        lambda values: (0.99, 0.8),
+    )
+    monkeypatch.setattr(
+        deep_statistics_v2.stats,
+        "normaltest",
+        lambda values: (20.0, 0.001),
+    )
+    monkeypatch.setattr(
+        deep_statistics_v2,
+        "_anderson_normality",
+        lambda values: {
+            "statistic": 2.0,
+            "p_value": 0.001,
+            "critical_5pct": None,
+            "method": "interpolate",
+        },
+    )
+
+    result = _normality(sample)
+
+    assert result["shapiro"]["p_value"] == pytest.approx(0.8)
+    assert result["dagostino"]["p_value"] == pytest.approx(0.001)
+    assert result["anderson"]["p_value"] == pytest.approx(0.001)
+    assert result["is_probably_normal"] is True

From 310e69534149a71944057b08d03cd4fb4d5dd7bb Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 10:43:01 +0530
Subject: [PATCH 409/439] test: support Anderson contracts across SciPy bounds

---
 tests/test_anderson_compatibility.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/tests/test_anderson_compatibility.py b/tests/test_anderson_compatibility.py
index ac239aa..0571f53 100644
--- a/tests/test_anderson_compatibility.py
+++ b/tests/test_anderson_compatibility.py
@@ -1,3 +1,4 @@
+import inspect
 from types import SimpleNamespace
 import warnings
 
@@ -9,7 +10,7 @@
 from framevitals.deep_statistics_v2 import _anderson_normality, _normality
 
 
-def test_anderson_modern_api_returns_pvalue_without_futurewarning():
+def test_anderson_supported_api_is_warning_free():
     rng = np.random.default_rng(42)
     sample = pd.Series(rng.normal(size=256))
 
@@ -18,10 +19,16 @@ def test_anderson_modern_api_returns_pvalue_without_futurewarning():
         result = _normality(sample)
 
     anderson = result["anderson"]
-    assert anderson["method"] == "interpolate"
     assert anderson["statistic"] is not None
-    assert anderson["critical_5pct"] is None
-    assert 0.0 <= anderson["p_value"] <= 1.0
+
+    if "method" in inspect.signature(deep_statistics_v2.stats.anderson).parameters:
+        assert anderson["method"] == "interpolate"
+        assert anderson["critical_5pct"] is None
+        assert 0.0 <= anderson["p_value"] <= 1.0
+    else:
+        assert anderson["method"] == "legacy_critical_values"
+        assert anderson["p_value"] is None
+        assert anderson["critical_5pct"] is not None
 
 
 def test_anderson_legacy_api_falls_back_to_critical_values(monkeypatch):

From cd9523a779011ce9356bd053a352da7f2cbef555 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 10:49:50 +0530
Subject: [PATCH 410/439] bench: add extensive 2.5B-cell physical stress matrix

---
 .../workflows/extensive-scale-benchmark.yml   | 269 ++++++++++++++++++
 1 file changed, 269 insertions(+)
 create mode 100644 .github/workflows/extensive-scale-benchmark.yml

diff --git a/.github/workflows/extensive-scale-benchmark.yml b/.github/workflows/extensive-scale-benchmark.yml
new file mode 100644
index 0000000..7819e7b
--- /dev/null
+++ b/.github/workflows/extensive-scale-benchmark.yml
@@ -0,0 +1,269 @@
+name: Extensive Physical Scale Benchmark
+
+on:
+  push:
+    branches: ["develop/august"]
+    paths:
+      - ".github/workflows/extensive-scale-benchmark.yml"
+      - "benchmarks/benchmark_real_wide_parquet.py"
+      - "src/framevitals/**"
+      - "rust/**"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  stress:
+    name: ${{ matrix.label }} — native vs fallback full pipeline
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: tall-250k-x-10k
+            rows: 250000
+            columns: 10000
+            row_group_rows: 10000
+          - label: wide-100k-x-25k
+            rows: 100000
+            columns: 25000
+            row_group_rows: 10000
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Configure Rust
+        run: |
+          rustup default stable
+          rustc --version
+          cargo --version
+
+      - name: Show runner capacity
+        run: |
+          nproc
+          free -h
+          df -h .
+
+      - name: Install benchmark and native build runtime
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[arrow]"
+          pip install "maturin>=1.14,<2"
+
+      - name: Build and install FrameVitals native engine
+        run: |
+          maturin build \
+            --release \
+            --manifest-path rust/framevitals-py/Cargo.toml \
+            --interpreter python \
+            --out dist-native
+          pip install --force-reinstall --no-deps dist-native/*.whl
+          FRAMEVITALS_BACKEND=rust python - <<'PY'
+          from framevitals.backends import backend_status
+
+          status = backend_status()
+          assert status["native_available"] is True, status
+          assert status["selected"] == "rust", status
+          print(status)
+          PY
+
+      - name: Generate physical Parquet workload
+        run: |
+          python benchmarks/benchmark_real_wide_parquet.py \
+            --generate-only \
+            --dataset "${{ matrix.label }}.parquet" \
+            --rows ${{ matrix.rows }} \
+            --columns ${{ matrix.columns }} \
+            --row-group-rows ${{ matrix.row_group_rows }} \
+            --output "generation-${{ matrix.label }}.json"
+          ls -lh "${{ matrix.label }}.parquet"
+          df -h .
+
+      - name: Run native Rust full analysis in all modes
+        env:
+          FRAMEVITALS_BACKEND: rust
+        run: |
+          for mode in quick standard deep research; do
+            echo "=== rust / ${{ matrix.label }} / ${mode} ==="
+            python benchmarks/benchmark_real_wide_parquet.py \
+              --dataset "${{ matrix.label }}.parquet" \
+              --mode "${mode}" \
+              --output "${{ matrix.label }}-rust-${mode}.json"
+          done
+
+      - name: Run NumPy fallback full analysis in all modes
+        env:
+          FRAMEVITALS_BACKEND: numpy
+        run: |
+          for mode in quick standard deep research; do
+            echo "=== numpy / ${{ matrix.label }} / ${mode} ==="
+            python benchmarks/benchmark_real_wide_parquet.py \
+              --dataset "${{ matrix.label }}.parquet" \
+              --mode "${mode}" \
+              --output "${{ matrix.label }}-numpy-${mode}.json"
+          done
+
+      - name: Validate routing, safety, accuracy, and backend parity
+        run: |
+          python - <<'PY'
+          import json
+          from pathlib import Path
+
+          label = "${{ matrix.label }}"
+          failures = []
+          payloads = {}
+
+          for backend in ("rust", "numpy"):
+              for mode in ("quick", "standard", "deep", "research"):
+                  path = Path(f"{label}-{backend}-{mode}.json")
+                  payload = json.loads(path.read_text())
+                  payloads[(backend, mode)] = payload
+
+                  observed = payload["backend"]["numeric_backend"]
+                  if observed != backend:
+                      failures.append(f"{backend}/{mode}: routed to {observed!r}")
+
+                  safety = payload["safety"]
+                  if safety["full_materialization"] is not False:
+                      failures.append(f"{backend}/{mode}: full materialization occurred")
+                  if safety["source_rows"] != ${{ matrix.rows }}:
+                      failures.append(f"{backend}/{mode}: wrong source row count")
+                  if safety["source_columns"] != ${{ matrix.columns }}:
+                      failures.append(f"{backend}/{mode}: wrong source column count")
+
+                  accuracy = payload["accuracy"]
+                  checked = accuracy["profiled_columns_checked"]
+                  if accuracy["missing_count_exact_matches"] != checked:
+                      failures.append(f"{backend}/{mode}: missing-count parity failed")
+                  if accuracy["numeric_count_exact_matches"] != checked:
+                      failures.append(f"{backend}/{mode}: numeric-count parity failed")
+                  if accuracy["numeric_minmax_exact_matches"] != checked:
+                      failures.append(f"{backend}/{mode}: min/max parity failed")
+                  if (accuracy["profile_mean_max_abs_error"] or 0.0) > 0.01:
+                      failures.append(
+                          f"{backend}/{mode}: profile mean max error "
+                          f"{accuracy['profile_mean_max_abs_error']}"
+                      )
+                  if accuracy["health_missing_abs_error"] > 0.02:
+                      failures.append(
+                          f"{backend}/{mode}: health missingness error "
+                          f"{accuracy['health_missing_abs_error']}"
+                      )
+
+                  deep_checked = accuracy["deep_numeric_columns_checked"]
+                  if deep_checked:
+                      if accuracy["deep_exact_once_columns"] != deep_checked:
+                          failures.append(f"{backend}/{mode}: exact-once reuse failed")
+                      if (accuracy["deep_mean_max_abs_error"] or 0.0) > 0.01:
+                          failures.append(
+                              f"{backend}/{mode}: deep exact mean error "
+                              f"{accuracy['deep_mean_max_abs_error']}"
+                          )
+
+          # The two backends should agree on the deterministic full-stream facts.
+          for mode in ("quick", "standard", "deep", "research"):
+              r = payloads[("rust", mode)]["accuracy"]
+              n = payloads[("numpy", mode)]["accuracy"]
+              for key in (
+                  "profiled_columns_checked",
+                  "missing_count_exact_matches",
+                  "numeric_count_exact_matches",
+                  "numeric_minmax_exact_matches",
+                  "deep_exact_once_columns",
+              ):
+                  if r[key] != n[key]:
+                      failures.append(
+                          f"{mode}: backend parity mismatch for {key}: {r[key]} != {n[key]}"
+                      )
+              if abs((r["profile_mean_max_abs_error"] or 0.0) - (n["profile_mean_max_abs_error"] or 0.0)) > 0.01:
+                  failures.append(f"{mode}: backend profile-mean accuracy diverged")
+
+          if failures:
+              raise SystemExit("\n".join(failures))
+          print("Extensive native/fallback full-pipeline checks passed.")
+          PY
+
+      - name: Publish stress summary
+        if: always()
+        run: |
+          python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
+          import json
+          from pathlib import Path
+
+          label = "${{ matrix.label }}"
+          gen_path = Path(f"generation-{label}.json")
+          if gen_path.exists():
+              g = json.loads(gen_path.read_text())
+              print(f"## {label}")
+              print()
+              print(f"- Physical shape: **{g['rows']:,} × {g['columns']:,}**")
+              print(f"- Logical cells: **{g['cells']:,}**")
+              print(f"- Dense int16 equivalent: **{g['dense_int16_raw_gb']:.2f} GB**")
+              print(f"- Parquet file size: **{g['file_size_mb']:.2f} MB**")
+              print(f"- Generation time: **{g['generation_seconds']:.3f}s**")
+              print()
+
+          print("| Backend | Mode | Wall time | Profiled cols | Sample rows | Profile max mean err | Deep max mean err |")
+          print("|---|---|---:|---:|---:|---:|---:|")
+          for backend in ("rust", "numpy"):
+              for mode in ("quick", "standard", "deep", "research"):
+                  path = Path(f"{label}-{backend}-{mode}.json")
+                  if not path.exists():
+                      print(f"| {backend} | {mode} | failed | - | - | - | - |")
+                      continue
+                  p = json.loads(path.read_text())
+                  s = p["safety"]
+                  a = p["accuracy"]
+                  print(
+                      f"| {backend} | {mode} | {p['analysis_seconds']:.3f}s | "
+                      f"{s['profiled_columns']} | {s['working_sample_rows']} | "
+                      f"{a['profile_mean_max_abs_error']} | {a['deep_mean_max_abs_error']} |"
+                  )
+          print()
+
+          for backend in ("rust", "numpy"):
+              for mode in ("quick", "standard", "deep", "research"):
+                  path = Path(f"{label}-{backend}-{mode}.json")
+                  if not path.exists():
+                      continue
+                  p = json.loads(path.read_text())
+                  print(f"### {backend} / {mode} stage timings")
+                  for name, ms in sorted(p.get("pipeline_timings_ms", {}).items()):
+                      if isinstance(ms, (int, float)):
+                          print(f"- {name}: {ms / 1000:.3f}s")
+                  print()
+          PY
+
+      - name: Upload benchmark results
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: framevitals-extensive-${{ matrix.label }}
+          path: |
+            generation-${{ matrix.label }}.json
+            ${{ matrix.label }}-rust-quick.json
+            ${{ matrix.label }}-rust-standard.json
+            ${{ matrix.label }}-rust-deep.json
+            ${{ matrix.label }}-rust-research.json
+            ${{ matrix.label }}-numpy-quick.json
+            ${{ matrix.label }}-numpy-standard.json
+            ${{ matrix.label }}-numpy-deep.json
+            ${{ matrix.label }}-numpy-research.json
+          if-no-files-found: ignore
+          retention-days: 30

From ff04c256459584001917fb6329c8b0353e104c0e Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 11:27:48 +0530
Subject: [PATCH 411/439] bench: add 5B-cell physical stress case

---
 .github/workflows/extensive-scale-benchmark.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/extensive-scale-benchmark.yml b/.github/workflows/extensive-scale-benchmark.yml
index 7819e7b..7a31e39 100644
--- a/.github/workflows/extensive-scale-benchmark.yml
+++ b/.github/workflows/extensive-scale-benchmark.yml
@@ -34,6 +34,10 @@ jobs:
             rows: 100000
             columns: 25000
             row_group_rows: 10000
+          - label: tall-500k-x-10k
+            rows: 500000
+            columns: 10000
+            row_group_rows: 10000
 
     steps:
       - name: Checkout repository

From 2f9bdc1ac23921e6e0c89508392d83d6ee3d4ec2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 11:29:34 +0530
Subject: [PATCH 412/439] test: add mixed statistical ground-truth validation

---
 .github/workflows/mixed-ground-truth.yml  | 133 ++++++++++++++++++
 tests/test_mixed_ground_truth_pipeline.py | 163 ++++++++++++++++++++++
 2 files changed, 296 insertions(+)
 create mode 100644 .github/workflows/mixed-ground-truth.yml
 create mode 100644 tests/test_mixed_ground_truth_pipeline.py

diff --git a/.github/workflows/mixed-ground-truth.yml b/.github/workflows/mixed-ground-truth.yml
new file mode 100644
index 0000000..1b0f8e1
--- /dev/null
+++ b/.github/workflows/mixed-ground-truth.yml
@@ -0,0 +1,133 @@
+name: Mixed Ground Truth Validation
+
+on:
+  push:
+    branches: ["develop/august"]
+    paths:
+      - ".github/workflows/mixed-ground-truth.yml"
+      - "tests/test_mixed_ground_truth_pipeline.py"
+      - "src/framevitals/**"
+      - "rust/**"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  validate:
+    name: native/fallback mixed statistical ground truth
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Configure Rust
+        run: |
+          rustup default stable
+          rustc --version
+          cargo --version
+
+      - name: Install test and Arrow dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev,arrow]"
+          pip install "maturin>=1.14,<2"
+
+      - name: Build and install FrameVitals native engine
+        run: |
+          maturin build \
+            --release \
+            --manifest-path rust/framevitals-py/Cargo.toml \
+            --interpreter python \
+            --out dist-native
+          pip install --force-reinstall --no-deps dist-native/*.whl
+          FRAMEVITALS_BACKEND=rust python - <<'PY'
+          from framevitals.backends import backend_status
+
+          status = backend_status()
+          assert status["native_available"] is True, status
+          assert status["selected"] == "rust", status
+          print(status)
+          PY
+
+      - name: NumPy fallback mixed ground-truth suite
+        env:
+          FRAMEVITALS_BACKEND: numpy
+        run: |
+          pytest -q \
+            tests/test_mixed_ground_truth_pipeline.py \
+            tests/test_parquet_streaming.py \
+            tests/test_target_intelligence.py \
+            tests/test_target_intelligence_v2.py \
+            tests/test_semantic_types.py \
+            tests/test_relationship_graph.py \
+            tests/test_fast_anomaly.py \
+            tests/test_deep_statistics.py \
+            --junitxml=mixed-ground-truth-numpy.xml
+
+      - name: Rust native mixed ground-truth suite
+        env:
+          FRAMEVITALS_BACKEND: rust
+        run: |
+          pytest -q \
+            tests/test_mixed_ground_truth_pipeline.py \
+            tests/test_parquet_streaming.py \
+            tests/test_target_intelligence.py \
+            tests/test_target_intelligence_v2.py \
+            tests/test_semantic_types.py \
+            tests/test_relationship_graph.py \
+            tests/test_fast_anomaly.py \
+            tests/test_deep_statistics.py \
+            --junitxml=mixed-ground-truth-rust.xml
+
+      - name: Publish validation summary
+        if: always()
+        run: |
+          python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
+          from pathlib import Path
+          import xml.etree.ElementTree as ET
+
+          print("## Mixed ground-truth validation")
+          print()
+          print("Coverage: mixed Parquet streaming, exact missingness/moments, quantile tolerance, categorical/date detection, target intelligence, semantic types, relationship discovery, anomaly recall, and backend parity through the same test contracts.")
+          print()
+          print("| Backend | Tests | Failures | Errors | Skipped |")
+          print("|---|---:|---:|---:|---:|")
+          for backend in ("numpy", "rust"):
+              path = Path(f"mixed-ground-truth-{backend}.xml")
+              if not path.exists():
+                  print(f"| {backend} | unavailable | - | - | - |")
+                  continue
+              root = ET.parse(path).getroot()
+              attrs = root.attrib
+              print(
+                  f"| {backend} | {attrs.get('tests', '?')} | {attrs.get('failures', '?')} | "
+                  f"{attrs.get('errors', '?')} | {attrs.get('skipped', '?')} |"
+              )
+          PY
+
+      - name: Upload validation reports
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: framevitals-mixed-ground-truth
+          path: |
+            mixed-ground-truth-numpy.xml
+            mixed-ground-truth-rust.xml
+          if-no-files-found: ignore
+          retention-days: 30
diff --git a/tests/test_mixed_ground_truth_pipeline.py b/tests/test_mixed_ground_truth_pipeline.py
new file mode 100644
index 0000000..9024944
--- /dev/null
+++ b/tests/test_mixed_ground_truth_pipeline.py
@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+import pytest
+
+pa = pytest.importorskip("pyarrow")
+pq = pytest.importorskip("pyarrow.parquet")
+
+import framevitals as fv
+from framevitals.fast_anomaly import fast_anomaly_scan
+from framevitals.relationship_graph import build_numeric_relationship_graph
+from framevitals.semantic_types import infer_semantic_types
+from framevitals.target_intelligence import run_target_intelligence
+
+
+def _mixed_frame(rows: int = 12_000) -> pd.DataFrame:
+    rng = np.random.default_rng(20260817)
+    linear = np.arange(rows, dtype=np.float64)
+    correlated = linear * 3.0 + 7.0
+    noisy_signal = linear * 0.25 + rng.normal(scale=5.0, size=rows)
+    normal = rng.normal(loc=4.0, scale=1.5, size=rows)
+    exponential = rng.exponential(scale=2.0, size=rows)
+
+    missing_numeric = normal.copy()
+    missing_numeric[::101] = np.nan
+
+    category = np.array(["alpha", "beta", "gamma", "delta"] * (rows // 4), dtype=object)
+    category[::97] = None
+
+    target = (linear % 10 >= 2).astype(np.int8)
+
+    return pd.DataFrame({
+        "linear": linear,
+        "correlated": correlated,
+        "noisy_signal": noisy_signal,
+        "normal": normal,
+        "exponential": exponential,
+        "missing_numeric": missing_numeric,
+        "category": category,
+        "event_time": pd.date_range("2026-01-01", periods=rows, freq="min"),
+        "target": target,
+    })
+
+
+def _write_mixed_parquet(path, rows: int = 12_000) -> pd.DataFrame:
+    frame = _mixed_frame(rows)
+    pq.write_table(pa.Table.from_pandas(frame, preserve_index=False), path, row_group_size=777)
+    return frame
+
+
+def test_mixed_parquet_profile_matches_ground_truth(tmp_path):
+    path = tmp_path / "mixed-ground-truth.parquet"
+    frame = _write_mixed_parquet(path)
+
+    result = fv.profile(path)
+
+    assert result["shape"] == {"rows": len(frame), "columns": len(frame.columns)}
+    assert result["streaming_metadata"]["enabled"] is True
+    assert result["streaming_metadata"]["full_materialization"] is False
+    assert result["missing_counts"]["missing_numeric"] == int(frame["missing_numeric"].isna().sum())
+    assert result["missing_counts"]["category"] == int(frame["category"].isna().sum())
+    assert result["numeric_summary"]["linear"]["mean"] == pytest.approx(
+        round(float(frame["linear"].mean()), 3)
+    )
+    assert result["numeric_summary"]["correlated"]["min"] == pytest.approx(7.0)
+    assert result["numeric_summary"]["correlated"]["max"] == pytest.approx(
+        float(frame["correlated"].max())
+    )
+    assert result["numeric_summary"]["linear"]["50%"] == pytest.approx(
+        float(frame["linear"].median()), rel=0.03
+    )
+    assert "category" in result["categorical_summary"]
+    assert "event_time" in result["date_columns"]
+
+
+def test_mixed_parquet_full_public_analysis_keeps_streaming_and_target_intelligence(tmp_path):
+    path = tmp_path / "mixed-analysis.parquet"
+    frame = _write_mixed_parquet(path)
+
+    result = fv.analyze(
+        path,
+        target="target",
+        mode="quick",
+        artifacts=False,
+        workers=2,
+    )
+
+    assert result["profile"]["shape"] == {"rows": len(frame), "columns": len(frame.columns)}
+    streaming = result["execution"]["streaming"]
+    assert streaming["enabled"] is True
+    assert streaming["full_materialization"] is False
+    assert streaming["single_full_source_profile_scan"] is True
+    assert result["target_intelligence"]["available"] is True
+    assert result["target_intelligence"]["target_column"] == "target"
+    assert result["target_intelligence"]["task_type"] == "classification"
+    assert 0 <= result["health"]["overall_score"] <= 100
+    assert 0 <= result["ml_readiness"]["score"] <= 100
+
+
+def test_known_numeric_relationship_is_found_without_dense_matrix():
+    frame = _mixed_frame(2_000)[["linear", "correlated", "noisy_signal", "normal"]]
+    result = build_numeric_relationship_graph(
+        frame,
+        max_sample_rows=512,
+        min_abs_correlation=0.98,
+    )
+
+    pairs = {
+        frozenset((edge["source"], edge["target"]))
+        for edge in result["edges"]
+    }
+    assert frozenset(("linear", "correlated")) in pairs
+    assert result["candidate_generation"]["candidate_pairs"] < result["candidate_generation"][
+        "total_possible_dense_pairs"
+    ]
+
+
+def test_injected_multivariate_anomalies_have_high_recall():
+    rng = np.random.default_rng(77)
+    rows = 3_000
+    matrix = rng.normal(size=(rows, 10))
+    injected = set(range(rows - 16, rows))
+    matrix[-16:] += 10.0
+    frame = pd.DataFrame(matrix, columns=[f"x{index}" for index in range(matrix.shape[1])])
+
+    result = fast_anomaly_scan(frame, contamination=0.02, top_k=30)
+    top_indices = {int(item["row_index"]) for item in result["top_rows"][:25]}
+
+    assert result["available"] is True
+    assert len(top_indices & injected) >= 12
+
+
+def test_target_intelligence_finds_numeric_and_categorical_leakage():
+    target = [0, 1] * 200
+    frame = pd.DataFrame({
+        "numeric_signal": [value * 10 + (index % 3) for index, value in enumerate(target)],
+        "segment": ["stay" if value == 0 else "leave" for value in target],
+        "noise": np.random.default_rng(9).normal(size=len(target)),
+        "target": target,
+    })
+
+    result = run_target_intelligence(frame, target_column="target")
+    associations = {item["feature"]: item for item in result["top_associations"]}
+    leakage = {item["feature"] for item in result["leakage"]["warnings"]}
+
+    assert result["task_type"] == "classification"
+    assert associations["numeric_signal"]["score"] > 0.9
+    assert associations["segment"]["score"] > 0.9
+    assert "segment" in leakage
+
+
+def test_semantic_type_ground_truth_for_sensitive_string_patterns():
+    cases = {
+        "email": pd.Series([f"user{index}@example.com" for index in range(40)]),
+        "url": pd.Series([f"https://example.com/{index}" for index in range(40)]),
+        "ip_address": pd.Series([f"10.0.0.{index + 1}" for index in range(40)]),
+    }
+
+    for expected, series in cases.items():
+        result = infer_semantic_types(series, max_samples=40)
+        assert result["primary"] == expected
+        assert result["candidates"][0]["confidence"] >= 0.7

From 12919d181fc5375773d65e5bea7d854a29175bb0 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 11:34:08 +0530
Subject: [PATCH 413/439] test: respect native full-stream categorical
 provenance

---
 tests/test_parquet_streaming.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
index 609f669..285a0b9 100644
--- a/tests/test_parquet_streaming.py
+++ b/tests/test_parquet_streaming.py
@@ -136,7 +136,18 @@ def test_large_parquet_retains_only_bounded_row_sample(tmp_path):
     assert streaming["full_materialization"] is False
     assert streaming["sample_rows"] == 50_000
     assert streaming["sample_strategy"] == "evenly_spaced_global_rows"
-    assert result["categorical_summary_metadata"]["sampled"] is True
+
+    categorical = result["categorical_summary_metadata"]
+    if categorical["native_full_stream_columns"]:
+        assert categorical["sampled"] is False
+        assert categorical["method"] == "native_full_stream_sketch"
+        assert "group" in categorical["native_full_stream_columns"]
+        assert categorical["sample_fallback_columns"] == []
+    else:
+        assert categorical["sampled"] is True
+        assert categorical["method"] == "evenly_spaced_row_sample"
+        assert "group" in categorical["sample_fallback_columns"]
+
     assert result["correlation_metadata"]["row_sampled"] is True
     assert result["duplicate_metadata"]["sampled"] is True
     assert result["missing_counts"]["value"] == int(frame["value"].isna().sum())

From 0320f25114800934a9c630a48c4835a381cf7f3f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 11:34:26 +0530
Subject: [PATCH 414/439] ci: rerun mixed validation for covered tests

---
 .github/workflows/mixed-ground-truth.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/mixed-ground-truth.yml b/.github/workflows/mixed-ground-truth.yml
index 1b0f8e1..56833cf 100644
--- a/.github/workflows/mixed-ground-truth.yml
+++ b/.github/workflows/mixed-ground-truth.yml
@@ -6,6 +6,13 @@ on:
     paths:
       - ".github/workflows/mixed-ground-truth.yml"
       - "tests/test_mixed_ground_truth_pipeline.py"
+      - "tests/test_parquet_streaming.py"
+      - "tests/test_target_intelligence.py"
+      - "tests/test_target_intelligence_v2.py"
+      - "tests/test_semantic_types.py"
+      - "tests/test_relationship_graph.py"
+      - "tests/test_fast_anomaly.py"
+      - "tests/test_deep_statistics.py"
       - "src/framevitals/**"
       - "rust/**"
   workflow_dispatch:

From 4679b76ab58aa58998931c679e4c6c9a34c664e7 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 11:45:43 +0530
Subject: [PATCH 415/439] fix: avoid periodic aliasing in bounded samples

---
 src/framevitals/execution.py         | 61 ++++++++++++++++++++++++----
 src/framevitals/streaming_profile.py | 25 +++++-------
 tests/test_sampling_strategy.py      | 41 +++++++++++++++++++
 3 files changed, 106 insertions(+), 21 deletions(-)
 create mode 100644 tests/test_sampling_strategy.py

diff --git a/src/framevitals/execution.py b/src/framevitals/execution.py
index 836de9b..1c951fa 100644
--- a/src/framevitals/execution.py
+++ b/src/framevitals/execution.py
@@ -19,6 +19,7 @@
 
 
 _VALID_MODES = {"quick", "standard", "deep", "research"}
+_SAMPLE_SEED = 0x9E3779B97F4A7C15
 
 # Full-stream profiling is valuable, but on ultra-wide sources scanning every
 # cell defeats the purpose of streaming. These budgets cap the number of source
@@ -75,6 +76,54 @@ def _bounded(requested: int, rows: int) -> int:
     return min(int(requested), int(rows))
 
 
+def _deterministic_stratified_positions(
+    rows: int,
+    target_rows: int,
+    *,
+    seed: int = _SAMPLE_SEED,
+) -> np.ndarray:
+    """Choose one deterministic pseudo-random row from each equal-width stratum.
+
+    Fixed evenly spaced samples can lock onto periodic structure. Stratified jitter
+    retains deterministic whole-dataset coverage while breaking that phase locking.
+    Positions are returned sorted, so temporal order remains available to callers
+    that need it, without allocating a permutation proportional to the source size.
+    """
+    rows = int(rows)
+    target_rows = int(target_rows)
+    count = min(rows, target_rows)
+    if count <= 0:
+        return np.empty(0, dtype=np.int64)
+    if count == rows:
+        return np.arange(rows, dtype=np.int64)
+    if count == 1:
+        return np.array([rows // 2], dtype=np.int64)
+
+    edges = np.fromiter(
+        ((index * rows) // count for index in range(count + 1)),
+        dtype=np.int64,
+        count=count + 1,
+    )
+    widths = (edges[1:] - edges[:-1]).astype(np.uint64)
+    indices = np.arange(count, dtype=np.uint64)
+
+    # SplitMix64-style deterministic mixing. uint64 overflow is intentional.
+    with np.errstate(over="ignore"):
+        mixed = indices + np.uint64(seed)
+        mixed = (mixed ^ (mixed >> np.uint64(30))) * np.uint64(0xBF58476D1CE4E5B9)
+        mixed = (mixed ^ (mixed >> np.uint64(27))) * np.uint64(0x94D049BB133111EB)
+        mixed = mixed ^ (mixed >> np.uint64(31))
+
+    offsets = (mixed % widths).astype(np.int64)
+    positions = edges[:-1] + offsets
+
+    # Keep full-range coverage as an explicit invariant while jittering the
+    # interior strata. This is useful for ordered/time-series diagnostics too.
+    positions[0] = 0
+    positions[-1] = rows - 1
+    return positions
+
+
 def derive_streaming_profile_column_limit(
     rows: int,
     columns: int,
@@ -243,20 +292,18 @@ def deterministic_sample_frame(
             "strategy": "full",
         }
 
-    # Evenly spaced positions are deterministic, preserve coverage across the
-    # source, and avoid allocating a random permutation proportional to n.
-    positions = np.linspace(0, source_rows - 1, num=max_rows, dtype=np.int64)
-    positions = np.unique(positions)
+    positions = _deterministic_stratified_positions(source_rows, max_rows)
     sampled = dataframe.iloc[positions]
     if not preserve_order:
-        # The positions are already ordered; this branch documents that callers
-        # may treat the sample as an unordered statistical sample.
+        # Positions stay sorted so time-aware callers can preserve ordering, while
+        # statistical callers receive a copy that is safe to mutate downstream.
         sampled = sampled.copy()
 
     return sampled, {
         "sampled": True,
         "source_rows": source_rows,
         "sample_rows": int(len(sampled)),
-        "strategy": "deterministic_evenly_spaced",
+        "strategy": "deterministic_stratified_jitter",
         "preserve_order": bool(preserve_order),
+        "seed": int(_SAMPLE_SEED),
     }
diff --git a/src/framevitals/streaming_profile.py b/src/framevitals/streaming_profile.py
index f85c988..35d32d4 100644
--- a/src/framevitals/streaming_profile.py
+++ b/src/framevitals/streaming_profile.py
@@ -1,10 +1,11 @@
 """Streaming profile construction for Arrow-capable dataset sources.
 
 The streaming path keeps full-row work to mergeable column state and retains
-only a bounded, evenly spaced row sample for analyses that still require row
-relationships (duplicate estimation and correlation). Native builds consume
-Arrow RecordBatches directly for fused numeric profiling and Arrow UTF-8 buffers
-for full-file categorical sketches when those kernels are available.
+only a bounded, deterministic stratified-jitter row sample for analyses that
+still require row relationships (duplicate estimation and correlation). Native
+builds consume Arrow RecordBatches directly for fused numeric profiling and
+Arrow UTF-8 buffers for full-file categorical sketches when those kernels are
+available.
 """
 
 from __future__ import annotations
@@ -24,6 +25,7 @@
     numeric_state,
     resolve_numeric_backend,
 )
+from framevitals.execution import _deterministic_stratified_positions
 from framevitals.profiler import _bounded_correlations, series_to_dict
 from framevitals.sources import StreamingDatasetSource
 from framevitals.streaming_sketches import (
@@ -260,12 +262,7 @@ def _summary_from_python_state(
 
 
 def _sample_positions(rows: int, target_rows: int) -> np.ndarray:
-    count = min(rows, target_rows)
-    if count <= 0:
-        return np.empty(0, dtype=np.int64)
-    if count == rows:
-        return np.arange(rows, dtype=np.int64)
-    return np.unique(np.linspace(0, rows - 1, num=count, dtype=np.int64))
+    return _deterministic_stratified_positions(rows, target_rows)
 
 
 def _sample_batch(batch, positions: np.ndarray, offset: int):
@@ -287,7 +284,7 @@ def sample_streaming_source(
     batch_size: int = STREAM_BATCH_SIZE,
     columns: Sequence[str] | None = None,
 ) -> pd.DataFrame:
-    """Return an evenly spaced bounded row sample without full materialization."""
+    """Return a deterministic stratified-jitter sample without full materialization."""
     if sample_rows < 1:
         raise ValueError("sample_rows must be at least 1.")
     if batch_size < 1:
@@ -363,7 +360,7 @@ def _estimated_memory_usage(sample: pd.DataFrame, rows: int) -> tuple[float, dic
         estimated = False
     else:
         estimated_bytes = int(round(sample_bytes / len(sample) * rows))
-        method = "scaled_from_even_row_sample"
+        method = "scaled_from_stratified_jitter_sample"
         estimated = True
     return round(estimated_bytes / (1024 * 1024), 3), {
         "method": method,
@@ -638,7 +635,7 @@ def build_streaming_profile(
             else "native_full_stream_sketch_with_sample_fallback"
         )
     else:
-        categorical_method = "exact" if len(sample) == rows else "evenly_spaced_row_sample"
+        categorical_method = "exact" if len(sample) == rows else "stratified_jitter_row_sample"
     categorical_metadata = {
         "method": categorical_method,
         "sampled": bool(sample_categorical_columns and len(sample) < rows),
@@ -708,7 +705,7 @@ def build_streaming_profile(
                 effective_batch_size < int(batch_size)
                 or effective_sample_rows < int(sample_rows)
             ),
-            "sample_strategy": "evenly_spaced_global_rows",
+            "sample_strategy": "stratified_jitter_global_rows",
             "numeric_backend": selected_backend,
             "native_arrow_fused_numeric": bool(native_arrow_fused),
             "numpy_full_stream_quantile_sketches": bool(use_numpy_quantile_sketches),
diff --git a/tests/test_sampling_strategy.py b/tests/test_sampling_strategy.py
new file mode 100644
index 0000000..70e6056
--- /dev/null
+++ b/tests/test_sampling_strategy.py
@@ -0,0 +1,41 @@
+import numpy as np
+import pandas as pd
+
+from framevitals.execution import (
+    _deterministic_stratified_positions,
+    deterministic_sample_frame,
+)
+from framevitals.streaming_profile import _sample_positions
+
+
+def test_stratified_positions_are_reproducible_sorted_and_cover_full_range():
+    first = _deterministic_stratified_positions(10_000, 1_000)
+    second = _deterministic_stratified_positions(10_000, 1_000)
+
+    assert np.array_equal(first, second)
+    assert len(first) == 1_000
+    assert first[0] == 0
+    assert first[-1] == 9_999
+    assert np.all(np.diff(first) > 0)
+
+
+def test_stratified_jitter_breaks_periodic_phase_locking():
+    rows = 5_000
+    period = 51
+    frame = pd.DataFrame({"periodic_spike": (np.arange(rows) % period == 0).astype(float)})
+
+    sampled, metadata = deterministic_sample_frame(frame, 50)
+
+    source_rate = float(frame["periodic_spike"].mean())
+    sampled_rate = float(sampled["periodic_spike"].mean())
+    assert abs(sampled_rate - source_rate) < 0.15
+    assert metadata["strategy"] == "deterministic_stratified_jitter"
+    assert metadata["sample_rows"] == 50
+    assert metadata["seed"] > 0
+
+
+def test_streaming_and_materialized_paths_share_sampling_positions():
+    expected = _deterministic_stratified_positions(20_000, 1_337)
+    actual = _sample_positions(20_000, 1_337)
+
+    assert np.array_equal(actual, expected)

From b2c8aff513491efeefd5ce8fe7e76822a6e20d15 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 11:50:21 +0530
Subject: [PATCH 416/439] build: ship native wheels alongside portable fallback

---
 .github/workflows/package.yml |  97 ++++++++++++++++------
 .github/workflows/release.yml | 148 +++++++++++++++++++++++++++++-----
 2 files changed, 201 insertions(+), 44 deletions(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index e28d687..8ac5431 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -27,55 +27,106 @@ jobs:
         with:
           python-version: "3.12"
 
+      - name: Configure Rust
+        run: |
+          rustup default stable
+          rustc --version
+          cargo --version
+
       - name: Install build tools
         run: |
           python -m pip install --upgrade pip
-          pip install build twine
+          pip install build twine "maturin==1.14.1"
+
+      - name: Build portable fallback distribution
+        run: python -m build --outdir dist-fallback
 
-      - name: Build package
-        run: python -m build
+      - name: Build native wheel
+        run: |
+          maturin build \
+            --release \
+            --manifest-path rust/framevitals-py/Cargo.toml \
+            --interpreter python \
+            --out dist-native \
+            --compatibility pypi
 
-      - name: Verify wheel contents
+      - name: Verify fallback and native wheel contents
         run: |
           python - <<'PY'
           from pathlib import Path
           import zipfile
 
-          wheels = list(Path("dist").glob("*.whl"))
-          assert len(wheels) == 1, f"Expected exactly one wheel, found {len(wheels)}"
+          fallback = list(Path("dist-fallback").glob("*.whl"))
+          native = list(Path("dist-native").glob("*.whl"))
+          assert len(fallback) == 1, f"Expected one fallback wheel, found {len(fallback)}"
+          assert len(native) == 1, f"Expected one native wheel, found {len(native)}"
 
-          wheel = wheels[0]
-          with zipfile.ZipFile(wheel) as archive:
-              names = archive.namelist()
+          def names(path: Path) -> list[str]:
+              with zipfile.ZipFile(path) as archive:
+                  return archive.namelist()
 
-          assert any(name.startswith("framevitals/") for name in names), (
-              "framevitals package missing from wheel"
-          )
-          assert "framevitals/py.typed" in names, (
-              "PEP 561 marker missing from wheel"
-          )
-          assert not any(name.startswith("modules/") for name in names), (
-              "legacy modules package leaked into wheel"
-          )
-          print(f"Verified distribution boundary: {wheel.name}")
+          fallback_names = names(fallback[0])
+          native_names = names(native[0])
+
+          for wheel, wheel_names in ((fallback[0], fallback_names), (native[0], native_names)):
+              assert any(name.startswith("framevitals/") for name in wheel_names), (
+                  f"framevitals package missing from {wheel.name}"
+              )
+              assert "framevitals/py.typed" in wheel_names, (
+                  f"PEP 561 marker missing from {wheel.name}"
+              )
+              assert not any(name.startswith("modules/") for name in wheel_names), (
+                  f"legacy modules package leaked into {wheel.name}"
+              )
+
+          native_suffixes = (".so", ".pyd", ".dylib")
+          assert not any(
+              name.startswith("framevitals/_native") and name.endswith(native_suffixes)
+              for name in fallback_names
+          ), "Portable fallback unexpectedly contains a native extension"
+          assert any(
+              name.startswith("framevitals/_native") and name.endswith(native_suffixes)
+              for name in native_names
+          ), "Native wheel does not contain framevitals._native"
+
+          print(f"Portable fallback: {fallback[0].name}")
+          print(f"Native wheel: {native[0].name}")
           PY
 
       - name: Validate distributions
-        run: python -m twine check dist/*
+        run: twine check dist-fallback/* dist-native/*
 
-      - name: Smoke-test built wheel
+      - name: Verify pip prefers the native wheel when compatible
         run: |
+          mkdir -p dist-install
+          cp dist-fallback/*.whl dist-install/
+          cp dist-native/*.whl dist-install/
           python -m venv /tmp/framevitals-wheel-smoke
-          /tmp/framevitals-wheel-smoke/bin/python -m pip install --no-deps dist/*.whl
+          /tmp/framevitals-wheel-smoke/bin/python -m pip install --upgrade pip
+          VERSION=$(python - <<'PY'
+          import tomllib
+          with open("pyproject.toml", "rb") as handle:
+              print(tomllib.load(handle)["project"]["version"])
+          PY
+          )
           cd /tmp
+          /tmp/framevitals-wheel-smoke/bin/python -m pip install \
+            --no-index \
+            --find-links "$GITHUB_WORKSPACE/dist-install" \
+            --no-deps \
+            "framevitals==$VERSION"
           /tmp/framevitals-wheel-smoke/bin/python - <<'PY'
           from importlib.metadata import version
           import framevitals
+          from framevitals.backends import backend_status
 
           installed = version("framevitals")
           assert installed == framevitals.__version__, (
               f"metadata version {installed} != package version {framevitals.__version__}"
           )
-          print(f"Imported FrameVitals {installed} from built wheel")
+          status = backend_status()
+          assert status["native_available"] is True, status
+          assert status["selected"] == "rust", status
+          print(f"Imported FrameVitals {installed} with native backend: {status}")
           PY
           /tmp/framevitals-wheel-smoke/bin/framevitals --version
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 92c5999..dd19c76 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -8,8 +8,8 @@ permissions:
   contents: read
 
 jobs:
-  build:
-    name: Build distribution
+  build-fallback:
+    name: Build portable fallback
     runs-on: ubuntu-latest
 
     steps:
@@ -21,14 +21,14 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.x"
+          python-version: "3.12"
 
       - name: Install build tools
         run: |
           python -m pip install --upgrade pip
           python -m pip install build twine
 
-      - name: Verify release version
+      - name: Verify release version across Python and Rust packages
         env:
           RELEASE_TAG: ${{ github.event.release.tag_name }}
         run: |
@@ -43,32 +43,116 @@ jobs:
           release_version = tag[1:] if tag.startswith("v") else tag
           with Path("pyproject.toml").open("rb") as handle:
               project_version = tomllib.load(handle)["project"]["version"]
+          with Path("rust/framevitals-core/Cargo.toml").open("rb") as handle:
+              core_version = tomllib.load(handle)["package"]["version"]
+          with Path("rust/framevitals-py/Cargo.toml").open("rb") as handle:
+              bridge_version = tomllib.load(handle)["package"]["version"]
           package_version = framevitals.__version__
 
-          assert release_version == project_version, (
-              f"release tag {tag!r} != pyproject version {project_version!r}"
-          )
-          assert project_version == package_version, (
-              f"pyproject version {project_version!r} != package version {package_version!r}"
-          )
-          print(f"Release version verified: {project_version}")
+          versions = {
+              "release tag": release_version,
+              "pyproject": project_version,
+              "python package": package_version,
+              "rust core": core_version,
+              "rust bridge": bridge_version,
+          }
+          assert len(set(versions.values())) == 1, f"Release versions disagree: {versions}"
+          print(f"Release version verified across Python/Rust packages: {project_version}")
           PY
 
-      - name: Build distributions
-        run: python -m build
+      - name: Build portable fallback wheel and sdist
+        run: python -m build --outdir dist
 
-      - name: Validate distributions
+      - name: Validate fallback distributions
         run: python -m twine check dist/*
 
-      - name: Store distributions
-        uses: actions/upload-artifact@v5
+      - name: Store fallback distributions
+        uses: actions/upload-artifact@v6
         with:
-          name: python-package-distributions
+          name: python-package-fallback
           path: dist/
+          if-no-files-found: error
+
+  build-native:
+    name: Native wheel — ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: linux-x86_64
+            os: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+            manylinux: "2_28"
+          - name: macos-arm64
+            os: macos-latest
+            target: aarch64-apple-darwin
+            manylinux: "off"
+          - name: macos-x86_64
+            os: macos-15-intel
+            target: x86_64-apple-darwin
+            manylinux: "off"
+          - name: windows-x86_64
+            os: windows-latest
+            target: x86_64-pc-windows-msvc
+            manylinux: "off"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Python 3.11 ABI floor
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Build native abi3 wheel
+        uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
+        with:
+          command: build
+          maturin-version: v1.14.1
+          target: ${{ matrix.target }}
+          manylinux: ${{ matrix.manylinux }}
+          args: >-
+            --release
+            --manifest-path rust/framevitals-py/Cargo.toml
+            --interpreter python
+            --out dist
+            --compatibility pypi
+
+      - name: Verify native wheel payload
+        shell: bash
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+          import zipfile
+
+          wheels = list(Path("dist").glob("*.whl"))
+          assert len(wheels) == 1, f"Expected one native wheel, found {len(wheels)}"
+          wheel = wheels[0]
+          with zipfile.ZipFile(wheel) as archive:
+              names = archive.namelist()
+          suffixes = (".so", ".pyd", ".dylib")
+          assert "framevitals/py.typed" in names, f"PEP 561 marker missing from {wheel.name}"
+          assert any(
+              name.startswith("framevitals/_native") and name.endswith(suffixes)
+              for name in names
+          ), f"Native extension missing from {wheel.name}"
+          print(f"Verified native distribution: {wheel.name}")
+          PY
+
+      - name: Store native wheel
+        uses: actions/upload-artifact@v6
+        with:
+          name: python-package-native-${{ matrix.name }}
+          path: dist/*.whl
+          if-no-files-found: error
 
   publish:
-    name: Publish distribution to PyPI
-    needs: build
+    name: Publish distributions to PyPI
+    needs: [build-fallback, build-native]
     runs-on: ubuntu-latest
     environment:
       name: pypi
@@ -78,11 +162,33 @@ jobs:
       contents: read
 
     steps:
-      - name: Download distributions
+      - name: Download all release distributions
         uses: actions/download-artifact@v6
         with:
-          name: python-package-distributions
+          pattern: python-package-*
           path: dist/
+          merge-multiple: true
+
+      - name: Show release payload
+        run: ls -lh dist/
+
+      - name: Validate complete release payload
+        run: |
+          python -m pip install --upgrade twine
+          python -m twine check dist/*
+          python - <<'PY'
+          from pathlib import Path
+
+          wheels = list(Path("dist").glob("*.whl"))
+          sdists = list(Path("dist").glob("*.tar.gz"))
+          native = [path for path in wheels if "none-any" not in path.name]
+          fallback = [path for path in wheels if "none-any" in path.name]
+
+          assert len(sdists) == 1, f"Expected one sdist, found {len(sdists)}"
+          assert len(fallback) == 1, f"Expected one portable fallback wheel, found {len(fallback)}"
+          assert len(native) == 4, f"Expected four native wheels, found {len(native)}: {native}"
+          print("Release payload contains one fallback wheel, one sdist, and four native wheels.")
+          PY
 
       - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1

From 7a7ed95348b791d2ce5085f4c790b00fe3d2f9be Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 11:53:44 +0530
Subject: [PATCH 417/439] test: install runtime deps for native wheel smoke

---
 .github/workflows/package.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index 8ac5431..b268fbf 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -103,6 +103,7 @@ jobs:
           cp dist-native/*.whl dist-install/
           python -m venv /tmp/framevitals-wheel-smoke
           /tmp/framevitals-wheel-smoke/bin/python -m pip install --upgrade pip
+          /tmp/framevitals-wheel-smoke/bin/python -m pip install numpy pandas
           VERSION=$(python - <<'PY'
           import tomllib
           with open("pyproject.toml", "rb") as handle:

From f4761cbf96d23ec727d08e6c3560f720d5992fa4 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 11:58:49 +0530
Subject: [PATCH 418/439] fix: align focused sampling provenance with
 stratified jitter

---
 src/framevitals/focused.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py
index abb2250..20e76a5 100644
--- a/src/framevitals/focused.py
+++ b/src/framevitals/focused.py
@@ -239,14 +239,14 @@ def statistics(
             "source_columns": source_columns,
             "sample_rows": int(len(sample)),
             "strategy": (
-                "streaming_evenly_spaced_global_rows"
+                "streaming_stratified_jitter_global_rows"
                 if sampled
                 else "full_stream_via_batches"
             ),
             "full_materialization": False,
             "reason": (
-                "Deep statistics ran on a bounded evenly spaced sample selected "
-                "directly from the streaming source."
+                "Deep statistics ran on a bounded deterministic stratified-jitter sample "
+                "selected directly from the streaming source."
                 if sampled
                 else "The streaming source fits within the deep-statistics execution budget."
             ),
@@ -355,14 +355,14 @@ def anomalies(
             "projected_columns": int(len(numeric_columns)),
             "sample_rows": int(len(sample)),
             "strategy": (
-                "streaming_evenly_spaced_numeric_projection"
+                "streaming_stratified_jitter_numeric_projection"
                 if sampled
                 else "full_stream_numeric_projection"
             ),
             "full_materialization": False,
             "reason": (
-                "Anomaly diagnostics ran on a bounded numeric projection selected "
-                "directly from the streaming source."
+                "Anomaly diagnostics ran on a bounded deterministic stratified-jitter "
+                "numeric projection selected directly from the streaming source."
                 if sampled
                 else "The streaming source fits within the anomaly execution budget."
             ),
@@ -436,13 +436,18 @@ def relationships(
         source_rows = int(metadata.rows or len(sample))
         source_columns = int(metadata.columns or len(sample.columns))
         sampled = source_rows > len(sample)
+        strategy = (
+            "streaming_stratified_jitter_global_rows"
+            if sampled
+            else "full_stream_via_batches"
+        )
         sample_metadata = payload.setdefault("sample", {})
         sample_metadata.update({
             "source_rows": source_rows,
             "sample_rows": int(len(sample)),
             "sampled": sampled,
             "full_materialization": False,
-            "strategy": "streaming_evenly_spaced_global_rows",
+            "strategy": strategy,
         })
         payload["source"] = metadata.to_dict()
         payload["streaming_source"] = True
@@ -455,7 +460,7 @@ def relationships(
             source_rows=source_rows,
             source_columns=source_columns,
             sample_rows=int(len(sample)),
-            strategy="streaming_evenly_spaced_global_rows",
+            strategy=strategy,
             components={
                 "numeric_projection": "schema_exact",
                 "relationship_candidates": (

From e47a18c83ed1234439c2fdd9c642392c12f0c5f1 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 11:59:24 +0530
Subject: [PATCH 419/439] test: assert stratified streaming sampling provenance

---
 tests/test_parquet_streaming.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py
index 285a0b9..17ce1da 100644
--- a/tests/test_parquet_streaming.py
+++ b/tests/test_parquet_streaming.py
@@ -135,7 +135,7 @@ def test_large_parquet_retains_only_bounded_row_sample(tmp_path):
     streaming = result["streaming_metadata"]
     assert streaming["full_materialization"] is False
     assert streaming["sample_rows"] == 50_000
-    assert streaming["sample_strategy"] == "evenly_spaced_global_rows"
+    assert streaming["sample_strategy"] == "stratified_jitter_global_rows"
 
     categorical = result["categorical_summary_metadata"]
     if categorical["native_full_stream_columns"]:
@@ -145,7 +145,7 @@ def test_large_parquet_retains_only_bounded_row_sample(tmp_path):
         assert categorical["sample_fallback_columns"] == []
     else:
         assert categorical["sampled"] is True
-        assert categorical["method"] == "evenly_spaced_row_sample"
+        assert categorical["method"] == "stratified_jitter_row_sample"
         assert "group" in categorical["sample_fallback_columns"]
 
     assert result["correlation_metadata"]["row_sampled"] is True
@@ -299,7 +299,7 @@ def fail_load(self):
     assert execution["projected_columns"] == 2
     assert execution["sample_rows"] == 5_000
     assert execution["sampled"] is True
-    assert execution["strategy"] == "streaming_evenly_spaced_numeric_projection"
+    assert execution["strategy"] == "streaming_stratified_jitter_numeric_projection"
     assert result["source"]["supports_projection"] is True
 
 
@@ -343,7 +343,7 @@ def fail_load(self):
     assert result["sample"]["sample_rows"] <= 256
     assert result["sample"]["sampled"] is True
     assert result["sample"]["full_materialization"] is False
-    assert result["sample"]["strategy"] == "streaming_evenly_spaced_global_rows"
+    assert result["sample"]["strategy"] == "streaming_stratified_jitter_global_rows"
     assert result["source"]["supports_projection"] is True
     edge_pairs = {(edge["source"], edge["target"]) for edge in result["edges"]}
     assert ("value", "other") in edge_pairs or ("other", "value") in edge_pairs

From 23e21f72d9b84a1cdf993cf6a40bc98ac4dc7efd Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 11:59:50 +0530
Subject: [PATCH 420/439] fix: avoid pandas string-dtype selection deprecation

---
 src/framevitals/fast_deep_statistics.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/framevitals/fast_deep_statistics.py b/src/framevitals/fast_deep_statistics.py
index eccd0d3..606eb77 100644
--- a/src/framevitals/fast_deep_statistics.py
+++ b/src/framevitals/fast_deep_statistics.py
@@ -214,15 +214,28 @@ def _numeric_bivariate(
     return numeric_pairs, binary_numeric_pairs, group_difference_tests
 
 
+def _categorical_columns(df: pd.DataFrame) -> list[str]:
+    """Select non-numeric categorical/string columns without dtype migration warnings."""
+    columns: list[str] = []
+    for column in df.columns:
+        dtype = df[column].dtype
+        if (
+            pd.api.types.is_object_dtype(dtype)
+            or pd.api.types.is_string_dtype(dtype)
+            or isinstance(dtype, pd.CategoricalDtype)
+            or pd.api.types.is_bool_dtype(dtype)
+        ):
+            columns.append(str(column))
+    return columns
+
+
 def run_fast_deep_statistics_v2(
     df: pd.DataFrame,
     max_pairs: int = 20,
 ) -> dict[str, Any]:
     """Run the v2 diagnostic battery with O(n) confidence intervals."""
     numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
-    categorical_cols = df.select_dtypes(
-        include=["object", "category", "bool"]
-    ).columns.tolist()
+    categorical_cols = _categorical_columns(df)
 
     categorical_view = (
         df.loc[:, categorical_cols]

From a73c94fdd80c7a1e1f39bdd88eaab11eba963c96 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 12:06:06 +0530
Subject: [PATCH 421/439] test: align statistics sampling provenance

---
 tests/test_statistics_streaming.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_statistics_streaming.py b/tests/test_statistics_streaming.py
index 9b30513..4b1e906 100644
--- a/tests/test_statistics_streaming.py
+++ b/tests/test_statistics_streaming.py
@@ -44,7 +44,7 @@ def fail_load(self):
     assert execution["source_columns"] == len(frame.columns)
     assert execution["sample_rows"] == 1_000
     assert execution["sampled"] is True
-    assert execution["strategy"] == "streaming_evenly_spaced_global_rows"
+    assert execution["strategy"] == "streaming_stratified_jitter_global_rows"
     assert execution["pair_budget"] == 2
     assert execution["source"]["format"] == "parquet"
     assert result["source"]["supports_streaming"] is True

From 30d04beddf330a6328a404bc0f4daf6b42f39cc0 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 12:08:33 +0530
Subject: [PATCH 422/439] feat: reuse exact full-stream skew and kurtosis

---
 rust/framevitals-core/src/lib.rs         | 121 ++++++++++++++++++++---
 rust/framevitals-py/src/lib.rs           |   5 +
 src/framevitals/analysis_state.py        |  83 +++++++++++++++-
 src/framevitals/backends.py              |  47 ++++++++-
 src/framevitals/streaming_exact_reuse.py |  23 +++--
 src/framevitals/streaming_pipeline.py    |   2 +-
 src/framevitals/streaming_profile.py     |  21 +++-
 tests/test_higher_moments.py             |  58 +++++++++++
 tests/test_streaming_exact_reuse.py      |  16 ++-
 9 files changed, 347 insertions(+), 29 deletions(-)
 create mode 100644 tests/test_higher_moments.py

diff --git a/rust/framevitals-core/src/lib.rs b/rust/framevitals-core/src/lib.rs
index f0c3abf..34d7b0a 100644
--- a/rust/framevitals-core/src/lib.rs
+++ b/rust/framevitals-core/src/lib.rs
@@ -19,6 +19,8 @@ pub struct NumericState {
     pub infinite: u64,
     pub mean: f64,
     pub m2: f64,
+    pub m3: f64,
+    pub m4: f64,
     pub minimum: Option<f64>,
     pub maximum: Option<f64>,
 }
@@ -31,6 +33,8 @@ impl Default for NumericState {
             infinite: 0,
             mean: 0.0,
             m2: 0.0,
+            m3: 0.0,
+            m4: 0.0,
             minimum: None,
             maximum: None,
         }
@@ -38,11 +42,12 @@ impl Default for NumericState {
 }
 
 impl NumericState {
-    /// Observe an optional numeric value using Welford's online algorithm.
+    /// Observe an optional numeric value using stable online central moments
+    /// through fourth order.
     ///
     /// `None` and NaN are counted as missing. Positive/negative infinity are
     /// counted separately and excluded from finite moments, matching the
-    /// current Python `NumericColumnState` semantics.
+    /// Python `NumericColumnState` semantics.
     pub fn observe(&mut self, value: Option<f64>) {
         let Some(value) = value else {
             self.missing += 1;
@@ -58,11 +63,24 @@ impl NumericState {
             return;
         }
 
+        let previous_count = self.count;
         self.count += 1;
+
+        let count = self.count as f64;
+        let previous_count_f = previous_count as f64;
         let delta = value - self.mean;
-        self.mean += delta / self.count as f64;
-        let delta2 = value - self.mean;
-        self.m2 += delta * delta2;
+        let delta_n = delta / count;
+        let delta_n2 = delta_n * delta_n;
+        let term1 = delta * delta_n * previous_count_f;
+
+        // M4/M3 depend on the previous lower-order moments, so update from the
+        // highest order down before advancing M2 and the mean.
+        self.m4 += term1 * delta_n2 * (count * count - 3.0 * count + 3.0)
+            + 6.0 * delta_n2 * self.m2
+            - 4.0 * delta_n * self.m3;
+        self.m3 += term1 * delta_n * (count - 2.0) - 3.0 * delta_n * self.m2;
+        self.m2 += term1;
+        self.mean += delta_n;
 
         self.minimum = Some(match self.minimum {
             Some(current) => current.min(value),
@@ -83,8 +101,8 @@ impl NumericState {
         state
     }
 
-    /// Merge independently computed partition states using the parallel
-    /// variance formula. No raw observations are required.
+    /// Merge independently computed partition states through fourth central
+    /// moment. No raw observations are required.
     #[must_use]
     pub fn merge(self, other: Self) -> Self {
         let missing = self.missing + other.missing;
@@ -110,9 +128,30 @@ impl NumericState {
         let total_f = total as f64;
         let left_f = self.count as f64;
         let right_f = other.count as f64;
+        let delta2 = delta * delta;
+        let delta3 = delta2 * delta;
+        let delta4 = delta2 * delta2;
+        let total2 = total_f * total_f;
+        let total3 = total2 * total_f;
 
         let mean = self.mean + delta * right_f / total_f;
-        let m2 = self.m2 + other.m2 + delta * delta * left_f * right_f / total_f;
+        let m2 = self.m2 + other.m2 + delta2 * left_f * right_f / total_f;
+        let m3 = self.m3
+            + other.m3
+            + delta3 * left_f * right_f * (left_f - right_f) / total2
+            + 3.0 * delta * (left_f * other.m2 - right_f * self.m2) / total_f;
+        let m4 = self.m4
+            + other.m4
+            + delta4
+                * left_f
+                * right_f
+                * (left_f * left_f - left_f * right_f + right_f * right_f)
+                / total3
+            + 6.0
+                * delta2
+                * (left_f * left_f * other.m2 + right_f * right_f * self.m2)
+                / total2
+            + 4.0 * delta * (left_f * other.m3 - right_f * self.m3) / total_f;
 
         Self {
             count: total,
@@ -120,6 +159,8 @@ impl NumericState {
             infinite,
             mean,
             m2,
+            m3,
+            m4,
             minimum: match (self.minimum, other.minimum) {
                 (Some(left), Some(right)) => Some(left.min(right)),
                 (left @ Some(_), None) => left,
@@ -148,6 +189,33 @@ impl NumericState {
     pub fn standard_deviation(&self) -> Option<f64> {
         self.variance().map(f64::sqrt)
     }
+
+    /// Bias-corrected Fisher-Pearson sample skewness, matching pandas/SciPy
+    /// `bias=False` semantics used by FrameVitals' deep-statistics report.
+    #[must_use]
+    pub fn skewness(&self) -> Option<f64> {
+        if self.count < 3 || self.m2 <= 0.0 {
+            return None;
+        }
+        let n = self.count as f64;
+        let population_skew = n.sqrt() * self.m3 / self.m2.powf(1.5);
+        Some((n * (n - 1.0)).sqrt() / (n - 2.0) * population_skew)
+    }
+
+    /// Bias-corrected Fisher excess kurtosis, matching pandas/SciPy
+    /// `fisher=True, bias=False` semantics.
+    #[must_use]
+    pub fn excess_kurtosis(&self) -> Option<f64> {
+        if self.count < 4 || self.m2 <= 0.0 {
+            return None;
+        }
+        let n = self.count as f64;
+        let population_excess = n * self.m4 / (self.m2 * self.m2) - 3.0;
+        Some(
+            (n - 1.0) / ((n - 2.0) * (n - 3.0))
+                * ((n + 1.0) * population_excess + 6.0),
+        )
+    }
 }
 
 #[cfg(test)]
@@ -156,7 +224,7 @@ mod tests {
 
     fn assert_close(left: f64, right: f64) {
         let scale = left.abs().max(right.abs()).max(1.0);
-        assert!((left - right).abs() <= 1e-12 * scale, "{left} != {right}");
+        assert!((left - right).abs() <= 1e-10 * scale, "{left} != {right}");
     }
 
     #[test]
@@ -178,16 +246,19 @@ mod tests {
         assert_eq!(state.maximum, Some(4.0));
         assert_close(state.mean, 7.0 / 3.0);
         assert_close(state.variance().unwrap(), 7.0 / 3.0);
+        assert!(state.skewness().is_some());
+        assert!(state.excess_kurtosis().is_none());
     }
 
     #[test]
-    fn partition_merge_matches_single_pass() {
+    fn partition_merge_matches_single_pass_through_fourth_moment() {
         let values: Vec<Option<f64>> = (0..10_000)
             .map(|value| {
                 if value % 97 == 0 {
                     None
                 } else {
-                    Some(value as f64 * 0.25 - 100.0)
+                    let x = value as f64 * 0.025 - 100.0;
+                    Some(x * x.signum() + (value % 11) as f64)
                 }
             })
             .collect();
@@ -205,7 +276,29 @@ mod tests {
         assert_eq!(merged.maximum, full.maximum);
         assert_close(merged.mean, full.mean);
         assert_close(merged.m2, full.m2);
+        assert_close(merged.m3, full.m3);
+        assert_close(merged.m4, full.m4);
         assert_close(merged.variance().unwrap(), full.variance().unwrap());
+        assert_close(merged.skewness().unwrap(), full.skewness().unwrap());
+        assert_close(
+            merged.excess_kurtosis().unwrap(),
+            full.excess_kurtosis().unwrap(),
+        );
+    }
+
+    #[test]
+    fn known_shape_statistics_match_reference_values() {
+        let state = NumericState::from_values(&[
+            Some(1.0),
+            Some(2.0),
+            Some(2.0),
+            Some(3.0),
+            Some(9.0),
+            Some(12.0),
+        ]);
+
+        assert_close(state.skewness().unwrap(), 1.069_287_452_144_894_3);
+        assert_close(state.excess_kurtosis().unwrap(), -0.796_319_305_259_674_9);
     }
 
     #[test]
@@ -222,9 +315,11 @@ mod tests {
     }
 
     #[test]
-    fn constant_values_have_zero_sample_variance() {
-        let state = NumericState::from_values(&[Some(3.0), Some(3.0), Some(3.0)]);
+    fn constant_values_have_zero_variance_and_undefined_shape() {
+        let state = NumericState::from_values(&[Some(3.0), Some(3.0), Some(3.0), Some(3.0)]);
         assert_eq!(state.variance(), Some(0.0));
         assert_eq!(state.standard_deviation(), Some(0.0));
+        assert_eq!(state.skewness(), None);
+        assert_eq!(state.excess_kurtosis(), None);
     }
 }
diff --git a/rust/framevitals-py/src/lib.rs b/rust/framevitals-py/src/lib.rs
index 70db753..d8712c4 100644
--- a/rust/framevitals-py/src/lib.rs
+++ b/rust/framevitals-py/src/lib.rs
@@ -25,8 +25,13 @@ fn exact_state_dict<'py>(py: Python<'py>, state: &NumericState) -> PyResult<Boun
             None
         },
     )?;
+    payload.set_item("m2", state.m2)?;
+    payload.set_item("m3", state.m3)?;
+    payload.set_item("m4", state.m4)?;
     payload.set_item("variance", state.variance())?;
     payload.set_item("std", state.standard_deviation())?;
+    payload.set_item("skewness", state.skewness())?;
+    payload.set_item("kurtosis", state.excess_kurtosis())?;
     payload.set_item("minimum", state.minimum)?;
     payload.set_item("maximum", state.maximum)?;
     Ok(payload)
diff --git a/src/framevitals/analysis_state.py b/src/framevitals/analysis_state.py
index 6722faa..8ce5f0b 100644
--- a/src/framevitals/analysis_state.py
+++ b/src/framevitals/analysis_state.py
@@ -19,12 +19,14 @@
 
 @dataclass(slots=True)
 class NumericColumnState:
-    """Mergeable exact first/second-moment state for one numeric column."""
+    """Mergeable exact central-moment state through fourth order."""
 
     count: int = 0
     missing: int = 0
     mean: float = 0.0
     m2: float = 0.0
+    m3: float = 0.0
+    m4: float = 0.0
     minimum: float | None = None
     maximum: float | None = None
     infinite: int = 0
@@ -38,7 +40,16 @@ def from_series(cls, series: pd.Series) -> "NumericColumnState":
             count=count,
             missing=int(payload["missing"]),
             mean=float(payload["mean"]) if count else 0.0,
-            m2=(float(variance) * (count - 1) if variance is not None and count >= 2 else 0.0),
+            m2=float(
+                payload.get(
+                    "m2",
+                    float(variance) * (count - 1)
+                    if variance is not None and count >= 2
+                    else 0.0,
+                )
+            ),
+            m3=float(payload.get("m3", 0.0)),
+            m4=float(payload.get("m4", 0.0)),
             minimum=(
                 float(payload["minimum"])
                 if payload.get("minimum") is not None
@@ -53,13 +64,15 @@ def from_series(cls, series: pd.Series) -> "NumericColumnState":
         )
 
     def merge(self, other: "NumericColumnState") -> "NumericColumnState":
-        """Merge another partition using the parallel-variance formula."""
+        """Merge another partition through fourth central moment."""
         if other.count == 0:
             return NumericColumnState(
                 count=self.count,
                 missing=self.missing + other.missing,
                 mean=self.mean,
                 m2=self.m2,
+                m3=self.m3,
+                m4=self.m4,
                 minimum=self.minimum,
                 maximum=self.maximum,
                 infinite=self.infinite + other.infinite,
@@ -70,15 +83,46 @@ def merge(self, other: "NumericColumnState") -> "NumericColumnState":
                 missing=self.missing + other.missing,
                 mean=other.mean,
                 m2=other.m2,
+                m3=other.m3,
+                m4=other.m4,
                 minimum=other.minimum,
                 maximum=other.maximum,
                 infinite=self.infinite + other.infinite,
             )
 
         total = self.count + other.count
+        left = float(self.count)
+        right = float(other.count)
+        total_f = float(total)
         delta = other.mean - self.mean
-        mean = self.mean + delta * other.count / total
-        m2 = self.m2 + other.m2 + delta * delta * self.count * other.count / total
+        delta2 = delta * delta
+        delta3 = delta2 * delta
+        delta4 = delta2 * delta2
+        total2 = total_f * total_f
+        total3 = total2 * total_f
+
+        mean = self.mean + delta * right / total_f
+        m2 = self.m2 + other.m2 + delta2 * left * right / total_f
+        m3 = (
+            self.m3
+            + other.m3
+            + delta3 * left * right * (left - right) / total2
+            + 3.0 * delta * (left * other.m2 - right * self.m2) / total_f
+        )
+        m4 = (
+            self.m4
+            + other.m4
+            + delta4
+            * left
+            * right
+            * (left * left - left * right + right * right)
+            / total3
+            + 6.0
+            * delta2
+            * (left * left * other.m2 + right * right * self.m2)
+            / total2
+            + 4.0 * delta * (left * other.m3 - right * self.m3) / total_f
+        )
         minimum = min(
             value for value in (self.minimum, other.minimum) if value is not None
         )
@@ -90,6 +134,8 @@ def merge(self, other: "NumericColumnState") -> "NumericColumnState":
             missing=self.missing + other.missing,
             mean=float(mean),
             m2=float(m2),
+            m3=float(m3),
+            m4=float(m4),
             minimum=float(minimum),
             maximum=float(maximum),
             infinite=self.infinite + other.infinite,
@@ -106,6 +152,28 @@ def std(self) -> float | None:
         variance = self.variance
         return sqrt(variance) if variance is not None and variance >= 0 else None
 
+    @property
+    def skewness(self) -> float | None:
+        """Bias-corrected Fisher-Pearson sample skewness."""
+        if self.count < 3 or self.m2 <= 0:
+            return None
+        n = float(self.count)
+        population_skew = sqrt(n) * self.m3 / (self.m2 ** 1.5)
+        return sqrt(n * (n - 1.0)) / (n - 2.0) * population_skew
+
+    @property
+    def kurtosis(self) -> float | None:
+        """Bias-corrected Fisher excess kurtosis."""
+        if self.count < 4 or self.m2 <= 0:
+            return None
+        n = float(self.count)
+        population_excess = n * self.m4 / (self.m2 * self.m2) - 3.0
+        return (
+            (n - 1.0)
+            / ((n - 2.0) * (n - 3.0))
+            * ((n + 1.0) * population_excess + 6.0)
+        )
+
     def to_dict(self) -> dict[str, Any]:
         return {
             "count": self.count,
@@ -114,6 +182,11 @@ def to_dict(self) -> dict[str, Any]:
             "mean": self.mean if self.count else None,
             "variance": self.variance,
             "std": self.std,
+            "skewness": self.skewness,
+            "kurtosis": self.kurtosis,
+            "m2": self.m2,
+            "m3": self.m3,
+            "m4": self.m4,
             "minimum": self.minimum,
             "maximum": self.maximum,
         }
diff --git a/src/framevitals/backends.py b/src/framevitals/backends.py
index 951dbe7..282faa8 100644
--- a/src/framevitals/backends.py
+++ b/src/framevitals/backends.py
@@ -62,6 +62,33 @@ def _float64_array(values: pd.Series | np.ndarray | list[Any]) -> np.ndarray:
     return np.ascontiguousarray(array, dtype=np.float64)
 
 
+def _bias_corrected_shape(
+    count: int,
+    m2: float,
+    m3: float,
+    m4: float,
+) -> tuple[float | None, float | None]:
+    """Return pandas/SciPy-compatible sample skewness and excess kurtosis."""
+    if count < 3 or m2 <= 0.0:
+        skewness = None
+    else:
+        n = float(count)
+        population_skew = np.sqrt(n) * m3 / (m2 ** 1.5)
+        skewness = float(np.sqrt(n * (n - 1.0)) / (n - 2.0) * population_skew)
+
+    if count < 4 or m2 <= 0.0:
+        kurtosis = None
+    else:
+        n = float(count)
+        population_excess = n * m4 / (m2 * m2) - 3.0
+        kurtosis = float(
+            (n - 1.0)
+            / ((n - 2.0) * (n - 3.0))
+            * ((n + 1.0) * population_excess + 6.0)
+        )
+    return skewness, kurtosis
+
+
 def _numpy_numeric_state(array: np.ndarray) -> dict[str, Any]:
     missing = int(np.isnan(array).sum())
     infinite = int(np.isinf(array).sum())
@@ -76,12 +103,23 @@ def _numpy_numeric_state(array: np.ndarray) -> dict[str, Any]:
             "mean": None,
             "variance": None,
             "std": None,
+            "skewness": None,
+            "kurtosis": None,
+            "m2": 0.0,
+            "m3": 0.0,
+            "m4": 0.0,
             "minimum": None,
             "maximum": None,
         }
 
     mean = float(finite.mean())
-    variance = float(finite.var(ddof=1)) if finite.size >= 2 else None
+    centered = finite - mean
+    centered2 = centered * centered
+    m2 = float(np.sum(centered2))
+    m3 = float(np.dot(centered2, centered))
+    m4 = float(np.dot(centered2, centered2))
+    variance = m2 / (finite.size - 1) if finite.size >= 2 else None
+    skewness, kurtosis = _bias_corrected_shape(int(finite.size), m2, m3, m4)
     return {
         "backend": "numpy",
         "observations": int(array.size),
@@ -91,6 +129,11 @@ def _numpy_numeric_state(array: np.ndarray) -> dict[str, Any]:
         "mean": mean,
         "variance": variance,
         "std": float(np.sqrt(variance)) if variance is not None else None,
+        "skewness": skewness,
+        "kurtosis": kurtosis,
+        "m2": m2,
+        "m3": m3,
+        "m4": m4,
         "minimum": float(finite.min()),
         "maximum": float(finite.max()),
     }
@@ -101,7 +144,7 @@ def numeric_state(
     *,
     backend: BackendName | str | None = None,
 ) -> dict[str, Any]:
-    """Calculate exact first/second-moment state through the selected backend."""
+    """Calculate exact mergeable moments through fourth order."""
     array = _float64_array(values)
     selected = resolve_numeric_backend(backend)
     if selected == "rust":
diff --git a/src/framevitals/streaming_exact_reuse.py b/src/framevitals/streaming_exact_reuse.py
index acf5a48..53aea5b 100644
--- a/src/framevitals/streaming_exact_reuse.py
+++ b/src/framevitals/streaming_exact_reuse.py
@@ -11,6 +11,8 @@
 
 from typing import Any
 
+from framevitals.deep_statistics_v2 import _classify_kurtosis, _classify_skew
+
 
 _EXACT_NUMERIC_FIELD_MAP = {
     "count": "count",
@@ -18,16 +20,18 @@
     "std": "std",
     "min": "min",
     "max": "max",
+    "skewness": "skewness",
+    "kurtosis": "kurtosis",
 }
 
 
 def reuse_streaming_exact_statistics(payload: dict[str, Any]) -> dict[str, Any]:
     """Overlay full-stream numeric sufficient statistics onto deep diagnostics.
 
-    Quantiles, skew/kurtosis, outlier views, distribution fits, hypothesis tests,
-    confidence intervals, and bivariate tests remain bounded-sample diagnostics.
-    Only statistics that the streaming profile already computed over every
-    profiled source row are reused.
+    Quantiles, outlier views, distribution fits, hypothesis tests, confidence
+    intervals, and bivariate tests remain bounded-sample diagnostics. Exact
+    count/mean/std/min/max and central-moment shape statistics are reused from
+    the already-completed full-stream profile pass.
     """
     profile = payload.get("profile")
     deep = payload.get("deep_statistics_v2")
@@ -59,6 +63,15 @@ def reuse_streaming_exact_statistics(payload: dict[str, Any]) -> dict[str, Any]:
             sample_summary[target_field] = exact_summary[source_field]
             reused_fields.append(target_field)
 
+        if "skewness" in reused_fields:
+            sample_summary["skewness_label"] = _classify_skew(
+                sample_summary.get("skewness")
+            )
+        if "kurtosis" in reused_fields:
+            sample_summary["kurtosis_label"] = _classify_kurtosis(
+                sample_summary.get("kurtosis")
+            )
+
         if reused_fields:
             reused_columns += 1
             sample_summary["summary_provenance"] = {
@@ -71,8 +84,6 @@ def reuse_streaming_exact_statistics(payload: dict[str, Any]) -> dict[str, Any]:
                     "median",
                     "q3",
                     "iqr",
-                    "skewness",
-                    "kurtosis",
                     "outliers",
                     "normality",
                     "distribution_fit",
diff --git a/src/framevitals/streaming_pipeline.py b/src/framevitals/streaming_pipeline.py
index f49a27b..eb541a9 100644
--- a/src/framevitals/streaming_pipeline.py
+++ b/src/framevitals/streaming_pipeline.py
@@ -369,7 +369,7 @@ def run_streaming_analysis(
     )
 
     sample_rows = int(len(working_sample))
-    sample_strategy = "streaming_evenly_spaced_global_rows"
+    sample_strategy = "streaming_stratified_jitter_global_rows"
     for key in _BOUNDED_RESULT_KEYS:
         sample_payload[key] = _scope_bounded_result(
             sample_payload.get(key),
diff --git a/src/framevitals/streaming_profile.py b/src/framevitals/streaming_profile.py
index 35d32d4..5b77746 100644
--- a/src/framevitals/streaming_profile.py
+++ b/src/framevitals/streaming_profile.py
@@ -193,7 +193,16 @@ def _state_from_payload(payload: dict[str, Any]) -> NumericColumnState:
         count=count,
         missing=int(payload["missing"]),
         mean=float(payload["mean"]) if count else 0.0,
-        m2=(float(variance) * (count - 1) if variance is not None and count >= 2 else 0.0),
+        m2=float(
+            payload.get(
+                "m2",
+                float(variance) * (count - 1)
+                if variance is not None and count >= 2
+                else 0.0,
+            )
+        ),
+        m3=float(payload.get("m3", 0.0)),
+        m4=float(payload.get("m4", 0.0)),
         minimum=(
             float(payload["minimum"])
             if payload.get("minimum") is not None
@@ -225,6 +234,8 @@ def _summary_from_native_snapshot(payload: dict[str, Any]) -> dict[str, Any]:
         "50%": _round_optional(quantiles.get("p50")),
         "75%": _round_optional(quantiles.get("p75")),
         "max": _round_optional(payload.get("maximum")),
+        "skewness": _round_optional(payload.get("skewness"), 6),
+        "kurtosis": _round_optional(payload.get("kurtosis"), 6),
     }
 
 
@@ -258,6 +269,8 @@ def _summary_from_python_state(
         "50%": _round_optional(q50),
         "75%": _round_optional(q75),
         "max": _round_optional(state.maximum),
+        "skewness": _round_optional(state.skewness, 6),
+        "kurtosis": _round_optional(state.kurtosis, 6),
     }
 
 
@@ -554,6 +567,8 @@ def build_streaming_profile(
             "quantile_relative_accuracy": quantile_accuracy,
             "quantile_source": "full_stream_sketch",
             "finite_only_moments": True,
+            "higher_moments": "full_stream_exact",
+            "shape_statistics": ["skewness", "kurtosis"],
             "columns_profiled": len(numeric_cols),
             "raw_observations_retained": False,
         }
@@ -575,6 +590,8 @@ def build_streaming_profile(
                 "quantile_source": "full_stream_sketch",
                 "quantile_cell_budget": int(PYTHON_NUMERIC_SKETCH_CELL_BUDGET),
                 "finite_only_moments": True,
+                "higher_moments": "full_stream_exact",
+                "shape_statistics": ["skewness", "kurtosis"],
                 "columns_profiled": len(numeric_cols),
                 "raw_observations_retained": False,
             }
@@ -588,6 +605,8 @@ def build_streaming_profile(
                 "quantile_cell_budget": int(PYTHON_NUMERIC_SKETCH_CELL_BUDGET),
                 "quantile_sketch_skipped_for_cost": bool(numeric_cols),
                 "finite_only_moments": True,
+                "higher_moments": "full_stream_exact",
+                "shape_statistics": ["skewness", "kurtosis"],
                 "columns_profiled": len(numeric_cols),
                 "raw_observations_retained": False,
             }
diff --git a/tests/test_higher_moments.py b/tests/test_higher_moments.py
new file mode 100644
index 0000000..a6bf22a
--- /dev/null
+++ b/tests/test_higher_moments.py
@@ -0,0 +1,58 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from framevitals.analysis_state import NumericColumnState
+from framevitals.backends import numeric_state
+
+
+def _reference_series() -> pd.Series:
+    return pd.Series([1.0, 2.0, 2.0, 3.0, 9.0, 12.0, np.nan, np.inf])
+
+
+def test_numpy_numeric_state_matches_pandas_bias_corrected_shape():
+    series = _reference_series()
+    finite = series[np.isfinite(series.to_numpy(dtype="float64", na_value=np.nan))]
+
+    state = numeric_state(series, backend="numpy")
+
+    assert state["count"] == len(finite)
+    assert state["missing"] == 1
+    assert state["infinite"] == 1
+    assert state["skewness"] == pytest.approx(float(finite.skew()), abs=1e-12)
+    assert state["kurtosis"] == pytest.approx(float(finite.kurtosis()), abs=1e-12)
+    assert state["m3"] != 0.0
+    assert state["m4"] > 0.0
+
+
+def test_numeric_column_state_partition_merge_preserves_shape_statistics():
+    values = pd.Series(
+        np.concatenate([
+            np.linspace(-8.0, 3.0, 2_500),
+            np.linspace(4.0, 40.0, 1_500) ** 1.15,
+        ])
+    )
+    full = NumericColumnState.from_series(values)
+    partitions = [
+        NumericColumnState.from_series(chunk)
+        for chunk in np.array_split(values, 7)
+    ]
+    merged = NumericColumnState()
+    for partition in partitions:
+        merged = merged.merge(partition)
+
+    assert merged.count == full.count
+    assert merged.mean == pytest.approx(full.mean, rel=1e-12, abs=1e-12)
+    assert merged.m2 == pytest.approx(full.m2, rel=1e-10, abs=1e-10)
+    assert merged.m3 == pytest.approx(full.m3, rel=1e-10, abs=1e-10)
+    assert merged.m4 == pytest.approx(full.m4, rel=1e-10, abs=1e-10)
+    assert merged.skewness == pytest.approx(float(values.skew()), rel=1e-10, abs=1e-10)
+    assert merged.kurtosis == pytest.approx(float(values.kurtosis()), rel=1e-10, abs=1e-10)
+
+
+def test_constant_state_has_undefined_shape_statistics():
+    state = NumericColumnState.from_series(pd.Series([5.0, 5.0, 5.0, 5.0, 5.0]))
+
+    assert state.std == 0.0
+    assert state.skewness is None
+    assert state.kurtosis is None
diff --git a/tests/test_streaming_exact_reuse.py b/tests/test_streaming_exact_reuse.py
index 64623bb..a1822b4 100644
--- a/tests/test_streaming_exact_reuse.py
+++ b/tests/test_streaming_exact_reuse.py
@@ -14,11 +14,14 @@ def test_reuses_full_stream_moments_without_overwriting_sample_diagnostics():
                     "50%": 12.0,
                     "75%": 15.0,
                     "max": 30.0,
+                    "skewness": 1.25,
+                    "kurtosis": 2.0,
                 }
             },
             "numeric_summary_metadata": {
                 "backend": "rust",
                 "method": "native_streaming_accumulator",
+                "higher_moments": "full_stream_exact",
             },
             "streaming_metadata": {
                 "enabled": True,
@@ -35,6 +38,9 @@ def test_reuses_full_stream_moments_without_overwriting_sample_diagnostics():
                     "max": 200.0,
                     "median": 13.0,
                     "skewness": 0.4,
+                    "skewness_label": "Approximately Symmetric",
+                    "kurtosis": -2.0,
+                    "kurtosis_label": "Light-tailed",
                     "distribution_fit": {"available": True, "best_fit": {"name": "norm"}},
                 }
             },
@@ -51,7 +57,10 @@ def test_reuses_full_stream_moments_without_overwriting_sample_diagnostics():
     assert stats["min"] == -2.0
     assert stats["max"] == 30.0
     assert stats["median"] == 13.0
-    assert stats["skewness"] == 0.4
+    assert stats["skewness"] == 1.25
+    assert stats["skewness_label"] == "Highly Skewed"
+    assert stats["kurtosis"] == 2.0
+    assert stats["kurtosis_label"] == "Heavy-tailed"
     assert stats["distribution_fit"]["best_fit"]["name"] == "norm"
     assert stats["summary_provenance"]["backend"] == "rust"
     assert set(stats["summary_provenance"]["reused_exact_fields"]) == {
@@ -60,10 +69,15 @@ def test_reuses_full_stream_moments_without_overwriting_sample_diagnostics():
         "std",
         "min",
         "max",
+        "skewness",
+        "kurtosis",
     }
+    assert "skewness" not in stats["summary_provenance"]["sample_derived_fields"]
+    assert "kurtosis" not in stats["summary_provenance"]["sample_derived_fields"]
     reuse = result["deep_statistics_v2"]["execution"]["exact_once_reuse"]
     assert reuse["enabled"] is True
     assert reuse["columns_reused"] == 1
+    assert set(reuse["fields"]) >= {"skewness", "kurtosis"}
 
 
 def test_noop_when_deep_statistics_are_not_applicable():

From 32749365e3a0e45ef221814a54445bf49b8d853c Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 12:10:44 +0530
Subject: [PATCH 423/439] style: format higher-moment Rust state

---
 rust/framevitals-core/src/lib.rs | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/rust/framevitals-core/src/lib.rs b/rust/framevitals-core/src/lib.rs
index 34d7b0a..a3b3bb3 100644
--- a/rust/framevitals-core/src/lib.rs
+++ b/rust/framevitals-core/src/lib.rs
@@ -142,15 +142,9 @@ impl NumericState {
             + 3.0 * delta * (left_f * other.m2 - right_f * self.m2) / total_f;
         let m4 = self.m4
             + other.m4
-            + delta4
-                * left_f
-                * right_f
-                * (left_f * left_f - left_f * right_f + right_f * right_f)
+            + delta4 * left_f * right_f * (left_f * left_f - left_f * right_f + right_f * right_f)
                 / total3
-            + 6.0
-                * delta2
-                * (left_f * left_f * other.m2 + right_f * right_f * self.m2)
-                / total2
+            + 6.0 * delta2 * (left_f * left_f * other.m2 + right_f * right_f * self.m2) / total2
             + 4.0 * delta * (left_f * other.m3 - right_f * self.m3) / total_f;
 
         Self {
@@ -211,10 +205,7 @@ impl NumericState {
         }
         let n = self.count as f64;
         let population_excess = n * self.m4 / (self.m2 * self.m2) - 3.0;
-        Some(
-            (n - 1.0) / ((n - 2.0) * (n - 3.0))
-                * ((n + 1.0) * population_excess + 6.0),
-        )
+        Some((n - 1.0) / ((n - 2.0) * (n - 3.0)) * ((n + 1.0) * population_excess + 6.0))
     }
 }
 

From 132c8609c0b36a36ec328d8ee7462dd070863969 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 12:15:16 +0530
Subject: [PATCH 424/439] docs: preserve useful architecture planning before
 branch cleanup

---
 .../BENCHMARK_ACCEPTANCE_PLAN.md              | 203 +++++++++++
 .../architecture-proposal/CAPABILITY_PACKS.md | 328 ++++++++++++++++++
 docs/architecture-proposal/CLI_TUI_SPEC.md    | 326 +++++++++++++++++
 docs/architecture-proposal/DECISIONS.md       | 136 ++++++++
 .../IMPLEMENTATION_ROADMAP.md                 | 274 +++++++++++++++
 .../PROPOSED_MODULE_TREE.md                   | 256 ++++++++++++++
 docs/architecture-proposal/README.md          |  24 ++
 7 files changed, 1547 insertions(+)
 create mode 100644 docs/architecture-proposal/BENCHMARK_ACCEPTANCE_PLAN.md
 create mode 100644 docs/architecture-proposal/CAPABILITY_PACKS.md
 create mode 100644 docs/architecture-proposal/CLI_TUI_SPEC.md
 create mode 100644 docs/architecture-proposal/DECISIONS.md
 create mode 100644 docs/architecture-proposal/IMPLEMENTATION_ROADMAP.md
 create mode 100644 docs/architecture-proposal/PROPOSED_MODULE_TREE.md
 create mode 100644 docs/architecture-proposal/README.md

diff --git a/docs/architecture-proposal/BENCHMARK_ACCEPTANCE_PLAN.md b/docs/architecture-proposal/BENCHMARK_ACCEPTANCE_PLAN.md
new file mode 100644
index 0000000..f5c153d
--- /dev/null
+++ b/docs/architecture-proposal/BENCHMARK_ACCEPTANCE_PLAN.md
@@ -0,0 +1,203 @@
+# FrameVitals Benchmark and Acceptance Plan
+
+A comprehensive analysis library needs explicit gates for performance and correctness. This document proposes how new modules should be evaluated before they enter standard/deep/exhaustive presets.
+
+## Why this matters
+
+FrameVitals should not equate “more analysis” with “better analysis.” A new method can be valuable only if it:
+- produces useful additional evidence
+- behaves correctly on edge cases
+- has predictable resource cost
+- degrades gracefully
+- does not make ordinary analysis disproportionately slower
+
+## Benchmark dimensions
+
+Each candidate analysis should be measured across:
+
+### Correctness
+- expected findings on golden datasets
+- no false structural assumptions on unsupported data
+- stable JSON/schema output
+- deterministic results when seeded
+- correct handling of missing/infinite/mixed values
+
+### Runtime
+- stage wall time
+- percentage of total analysis time
+- cold vs warm/cache-aware time where relevant
+
+### Memory
+- peak resident memory where measurable
+- temporary dataframe/array copies
+- model/runtime memory for optional ML/DL features
+
+### Scalability
+- tall datasets
+- wide datasets
+- high-cardinality categorical data
+- long text columns
+- large numeric matrices
+
+### Reproducibility
+- package version
+- config hash
+- random seed
+- backend/runtime
+- optional pack versions
+- model ID/version/checksum
+
+## Proposed benchmark dataset families
+
+Synthetic/golden fixtures should cover at least:
+
+1. **Clean mixed tabular** — representative numeric + categorical + date columns.
+2. **Missingness-heavy** — MCAR-like, patterned, and column-dependent missingness.
+3. **Duplicate/key issues** — duplicate rows, duplicate candidate IDs, conflicting records.
+4. **High cardinality** — IDs, codes, near-unique categoricals.
+5. **Outlier/anomaly** — univariate and multivariate injected anomalies.
+6. **Drift numeric** — mean/variance/tail/shape changes.
+7. **Drift categorical** — frequency shifts, new/missing categories.
+8. **Target classification** — balanced and imbalanced labels, leakage and proxy leakage.
+9. **Target regression** — nonlinear and linear signal, leakage cases.
+10. **Time series** — regular/irregular timestamps, trend, seasonality, gaps, anomalies.
+11. **Text** — short categories, long text, duplicates, semantic shift where model packs are tested.
+12. **Adversarial dtype** — numeric-looking strings, mixed dates, booleans, infinities, malformed input.
+13. **Wide** — hundreds/thousands of columns with modest rows.
+14. **Tall** — large row counts with a small/medium number of columns.
+
+## Standard size classes
+
+Use named classes rather than hardcoding expectations around one machine:
+
+| Class | Example purpose |
+|---|---|
+| Tiny | unit/smoke behavior |
+| Small | interactive notebook/CLI usage |
+| Medium | common local analytics workload |
+| Large | sampling/streaming/backend strategy validation |
+| Wide | column-scaling validation |
+
+Exact row/column counts should be defined in the benchmark suite and can evolve as performance improves.
+
+## Per-analysis cost classes
+
+Every analysis registry entry should declare an expected cost class:
+
+- `tiny` — metadata/simple cached values
+- `low` — one lightweight vectorized pass
+- `medium` — correlations/statistical tests/model-like operations
+- `high` — ensemble, large pairwise relationships, expensive transforms
+- `very_high` — deep models, embeddings, exhaustive pairwise/search procedures
+
+The planner uses this declaration together with observed dimensions and configuration budgets.
+
+## Default-preset admission rules
+
+### Quick
+An analysis belongs in `quick` only if it is:
+- highly reliable
+- broadly applicable
+- low cost
+- useful without extensive interpretation
+
+### Standard
+An analysis belongs in `standard` when:
+- value is high for ordinary datasets
+- cost is bounded/predictable
+- it has strong fallback behavior
+
+### Deep
+An analysis belongs in `deep` when:
+- it adds meaningful evidence beyond standard
+- higher cost is justified
+- output is still interpretable
+
+### Exhaustive
+An analysis can be available in `exhaustive` when:
+- applicability is explicit
+- resource estimates exist
+- optional requirements are declared
+- failures can be isolated
+
+Being present in the codebase does not automatically qualify an analysis for any preset.
+
+## Regression policy
+
+Performance changes should be reviewed at the stage level rather than only using total runtime.
+
+Flag changes when:
+- an existing stage becomes materially slower without an intentional tradeoff
+- memory copies increase unexpectedly
+- a default preset begins running a previously optional expensive module
+- cache/context reuse regresses
+
+Do not enforce overly tight microbenchmark thresholds in noisy CI environments. Prefer broad regression bands, repeated samples, and trend reporting.
+
+## Model acceptance rules
+
+An optional ML/DL diagnostic should have a stronger acceptance bar.
+
+Before inclusion:
+1. define exactly which analytical question it answers
+2. compare against simpler deterministic/classical baselines
+3. measure incremental detection/value, not only model metrics
+4. measure CPU runtime and memory
+5. document GPU behavior if supported
+6. test deterministic seeds where possible
+7. document failure/fallback behavior
+8. verify model/runtime license compatibility
+9. record model/version metadata in results
+
+A deep model should not be enabled in the standard preset simply because it performs slightly better on one benchmark.
+
+## Report-quality tests
+
+Generated output is part of correctness.
+
+Test:
+- JSON serialization
+- HTML contains required sections
+- no missing/invalid links/assets for self-contained reports
+- terminal renderer handles narrow/non-color terminals
+- privacy mode does not expose raw sensitive examples
+- report generation does not re-run expensive analysis
+
+## Backend parity
+
+As Polars/PyArrow/Narwhals support grows, parity tests should focus on **semantic equivalence**, not byte-for-byte output.
+
+Examples:
+- same column semantic roles
+- same missing/duplicate counts
+- same contract verdicts
+- same drift severity class within numerical tolerance
+- same findings where algorithms are equivalent
+
+Backend-specific optimized approximations should clearly record their method.
+
+## Suggested benchmark command surface
+
+Future developer tooling could expose:
+
+```bash
+python -m benchmarks.run --suite core
+python -m benchmarks.run --suite drift
+python -m benchmarks.run --suite large
+python -m benchmarks.compare baseline.json candidate.json
+```
+
+A user-facing `framevitals benchmark` command is not necessary initially; these are primarily development/release tools.
+
+## Release gate summary
+
+Before a significant analysis capability moves into a default preset:
+- correctness fixtures pass
+- serialization/report tests pass
+- runtime/memory impact is understood
+- fallback behavior is tested
+- applicability is registered
+- documentation explains the result
+- no silent install/download behavior exists
+
+This keeps “complete and exhaustive” compatible with “fast and trustworthy.”
diff --git a/docs/architecture-proposal/CAPABILITY_PACKS.md b/docs/architecture-proposal/CAPABILITY_PACKS.md
new file mode 100644
index 0000000..820d2d9
--- /dev/null
+++ b/docs/architecture-proposal/CAPABILITY_PACKS.md
@@ -0,0 +1,328 @@
+# FrameVitals Capability Pack Specification
+
+This document defines how optional capabilities can grow without turning the base install into a huge dependency bundle.
+
+## Design goals
+
+- The base `framevitals` install must remain useful by itself.
+- Optional packs should add clearly defined capabilities.
+- Installation and enablement are separate states.
+- Model weights should not be bundled into PyPI wheels unless tiny and justified.
+- Deep-learning frameworks must stay optional.
+- Every optional capability must fail gracefully and preserve deterministic fallbacks when possible.
+
+## Proposed packs
+
+### `core`
+Installed by default.
+
+Scope:
+- CSV/TSV/basic tabular loading
+- structural profiling
+- data quality
+- health score
+- core statistics
+- contracts/validation
+- drift basics
+- result schema
+- standard CLI
+
+Goal: a strong data-analysis/data-health library even with no extras.
+
+---
+
+### `viz`
+Purpose: rendering and rich human-facing reports.
+
+Potential contents:
+- plotting dependencies
+- standalone HTML report helpers
+- PDF/chart rendering
+- notebook visual components
+
+CLI/TUI label: **Visual Reports**
+
+---
+
+### `excel`
+Purpose: XLS/XLSX ingestion.
+
+Potential contents:
+- `openpyxl`
+- `xlrd` where still required
+
+CLI/TUI label: **Excel Support**
+
+---
+
+### `ml`
+Purpose: classical ML-based diagnostics.
+
+Potential capabilities:
+- baseline models
+- feature importance
+- model diagnostics
+- Isolation Forest
+- LOF
+- ECOD/COPOD/HBOS or equivalent anomaly methods
+- optional gradient boosting diagnostics
+- optional SHAP-style explanation layer
+
+This pack is diagnostic, not a general AutoML system.
+
+CLI/TUI label: **ML Diagnostics**
+
+---
+
+### `deep`
+Purpose: optional nonlinear/deep diagnostics where they add real value.
+
+Potential capabilities:
+- MLP autoencoder anomaly detector
+- DeepSVDD-style anomaly detector
+- variational autoencoder experiment path
+- small temporal CNN/TCN models
+- optional tabular nonlinear baseline
+
+Framework policy:
+- do not require a deep-learning framework in the core package
+- choose one supported runtime initially rather than supporting everything
+- CPU must remain supported for small models where practical
+- GPU use should be opt-in/auto-detected and never required for ordinary FrameVitals analysis
+
+CLI/TUI label: **Deep Models**
+
+---
+
+### `text`
+Purpose: advanced text and semantic diagnostics.
+
+Potential capabilities:
+- embedding-based text drift
+- semantic similarity/grouping
+- advanced PII helpers
+- language detection
+- text quality/duplication signals
+
+Large embedding weights should be managed by the model registry, not bundled directly into the wheel.
+
+CLI/TUI label: **Advanced Text / NLP**
+
+---
+
+### `polars`
+Purpose: Polars input/backend support.
+
+Potential capabilities:
+- DataFrame/LazyFrame adapters
+- projection/predicate-aware analysis paths where practical
+- backend-native operations for large data
+
+CLI/TUI label: **Polars Backend**
+
+---
+
+### `arrow`
+Purpose: PyArrow and columnar I/O.
+
+Potential capabilities:
+- Arrow Table/RecordBatch support
+- Parquet scans
+- batch/streaming analysis paths
+- projection/filter support
+
+CLI/TUI label: **Arrow / Parquet Backend**
+
+---
+
+### `sql`
+Purpose: database/query-backed analysis.
+
+Potential capabilities:
+- DuckDB adapter
+- SQLAlchemy-compatible sources where useful
+- pushdown of lightweight aggregates/projections
+
+The initial goal should be analysis of query results and large local files rather than becoming a database management tool.
+
+CLI/TUI label: **SQL Adapters**
+
+---
+
+### `cloud`
+Purpose: remote object/file storage integration.
+
+Potential capabilities:
+- fsspec-based inputs
+- S3-compatible paths
+- selected cloud filesystem adapters
+
+Credentials must be supplied by the user's environment/provider tooling; FrameVitals should not invent its own secret store.
+
+CLI/TUI label: **Cloud Filesystems**
+
+---
+
+### `tui`
+Purpose: interactive terminal application if kept separate from core.
+
+Potential contents:
+- terminal UI framework
+- interactive tables/forms/progress views
+
+If the dependency footprint is small enough, this pack may eventually become part of the default install. Until then, the standard argparse CLI remains available without it.
+
+CLI/TUI label: **Interactive Terminal UI**
+
+---
+
+### `ai`
+Purpose: optional natural-language interpretation/agent features.
+
+Rules:
+- deterministic analysis remains the source of factual metrics
+- AI summarizes/interprets structured findings
+- no AI dependency required for normal analysis
+- provider-specific behavior stays isolated
+
+CLI/TUI label: **AI Interpretation**
+
+## Example extras layout
+
+Conceptually:
+
+```toml
+[project.optional-dependencies]
+viz = ["..."]
+excel = ["..."]
+ml = ["..."]
+deep = ["..."]
+text = ["..."]
+polars = ["..."]
+arrow = ["..."]
+sql = ["..."]
+cloud = ["..."]
+tui = ["..."]
+ai = ["..."]
+```
+
+The exact dependency choices should be benchmarked and reviewed before modifying `pyproject.toml`.
+
+## Capability registry
+
+FrameVitals should expose capability metadata independently from the Python packaging implementation.
+
+Example conceptual record:
+
+```python
+Capability(
+    id="deep",
+    name="Deep Models",
+    installed=False,
+    enabled=False,
+    extra="deep",
+    provides=[
+        "anomaly.autoencoder",
+        "anomaly.deep_svdd",
+        "timeseries.tcn",
+    ],
+    resource_class="high",
+)
+```
+
+This registry powers:
+- `framevitals addons list`
+- TUI install/toggle screens
+- planner applicability checks
+- `framevitals doctor`
+- error messages when an optional analysis was explicitly requested but is unavailable
+
+## Installed vs enabled
+
+These states must not be conflated.
+
+| Installed | Enabled | Meaning |
+|---|---|---|
+| No | No | capability unavailable |
+| Yes | No | dependency exists but planner will not use it automatically |
+| Yes | Yes | planner may use capability when applicable |
+| No | Yes | invalid state; config should warn and treat as unavailable |
+
+Users should be able to keep a pack installed but disabled for speed/reproducibility.
+
+## Model registry
+
+Model files should have a separate lifecycle from Python packages.
+
+Suggested cache organization:
+
+```text
+<framevitals-data-dir>/
+  models/
+    semantic-types-small/
+      1.0.0/
+        model...
+        manifest.json
+    temporal-anomaly-tiny/
+      1.0.0/
+        model...
+        manifest.json
+```
+
+Each manifest should include:
+- model ID/version
+- FrameVitals compatibility
+- task
+- framework/runtime
+- checksum
+- source URL
+- license
+- file sizes
+- expected input contract
+
+## Download policy
+
+Model download should occur only through explicit actions such as:
+
+```bash
+framevitals models install semantic-types-small
+```
+
+or the equivalent TUI **Download** button.
+
+Ordinary analysis can suggest a model but should not automatically download it.
+
+Example:
+
+```text
+Semantic type confidence is low for 3 columns.
+Optional model 'semantic-types-small' could provide a second opinion.
+[ Download & enable ] [ Ignore ]
+```
+
+Even there, user confirmation is required.
+
+## Reproducibility
+
+Every analysis result should record optional capability/model metadata when used:
+- pack ID/version
+- relevant dependency versions
+- model ID/version/checksum
+- backend
+- random seed
+- device (CPU/GPU)
+
+This makes deep/model-assisted diagnostics reproducible and explainable.
+
+## Dependency policy
+
+Before adding a package to any optional pack, evaluate:
+1. user value
+2. wheel size and install reliability
+3. Python/platform compatibility
+4. maintenance activity
+5. license compatibility
+6. startup/import cost
+7. whether FrameVitals can implement the needed behavior with an existing dependency instead
+
+A large dependency should not enter the base install merely because one analysis can use it.
diff --git a/docs/architecture-proposal/CLI_TUI_SPEC.md b/docs/architecture-proposal/CLI_TUI_SPEC.md
new file mode 100644
index 0000000..5cde425
--- /dev/null
+++ b/docs/architecture-proposal/CLI_TUI_SPEC.md
@@ -0,0 +1,326 @@
+# FrameVitals CLI / TUI Specification
+
+This document defines the intended command-line and interactive terminal experience for future FrameVitals releases.
+
+## Goals
+
+The CLI should satisfy two very different users without forcing either into the other's workflow:
+
+1. **Interactive user** — wants to explore configuration, install optional capabilities, select analyses, and inspect reports through a terminal UI.
+2. **Automation user** — wants stable commands, exit codes, JSON output, and no prompts for scripts/CI.
+
+The same underlying configuration and execution engine must power both experiences.
+
+## Entry behavior
+
+```bash
+framevitals
+```
+
+If an interactive TTY is available and the TUI capability is installed, open the FrameVitals terminal application.
+
+If the TUI is unavailable, show normal CLI help and a short message explaining how to install the interactive capability.
+
+Explicit subcommands must never unexpectedly open the TUI:
+
+```bash
+framevitals analyze data.csv
+framevitals compare reference.csv current.csv
+framevitals validate data.csv --contract contract.json
+```
+
+## Main navigation
+
+```text
+FrameVitals
+
+  Analyze Dataset
+  Compare Datasets
+  Validate Contract
+  Reports & Snapshots
+  Configuration
+  Add-ons
+  Models
+  Doctor
+  Help
+  Exit
+```
+
+## Analysis screen
+
+```text
+Dataset: /Users/me/data/customers.csv
+Target:  churn
+
+Preset
+( ) Quick   (*) Standard   ( ) Deep   ( ) Exhaustive   ( ) Custom
+
+Analysis categories
+[x] Structural profile
+[x] Data quality
+[x] Statistics
+[x] Relationships
+[x] ML readiness
+[x] Target intelligence
+[x] Drift / comparison       auto when reference exists
+[x] Time-series analysis     auto when applicable
+[x] Text analysis            auto when applicable
+[x] Privacy / PII checks
+[ ] Deep models              optional pack
+[ ] AI interpretation        optional pack
+
+Resources
+Backend       Auto
+Workers       4
+Memory limit  4 GB
+Time limit    120 s
+GPU           Auto
+Sampling      Adaptive
+
+[ Show Plan ] [ Run Analysis ] [ Save Profile ]
+```
+
+## Plan screen
+
+Before expensive runs, users should be able to see what FrameVitals intends to execute.
+
+```text
+Execution Plan
+
+RUN   structural_profile        essential     ~low
+RUN   missingness_analysis      essential     ~low
+RUN   relationship_matrix       high          ~medium
+RUN   anomaly_ensemble          high          ~medium
+SKIP  tcn_temporal_anomaly      not installed
+SKIP  text_embeddings           no applicable text column
+SKIP  target_leakage            no target selected
+
+Estimated class: medium
+Expected peak memory: < 2 GB
+
+[ Run ] [ Configure ] [ Back ]
+```
+
+Every skipped item should have an explicit reason.
+
+## Add-on manager
+
+Optional functionality must be easy to discover and install from the TUI.
+
+```text
+Add-ons
+
+Core
+[x] Core analysis              Built in
+[x] Contracts                  Built in
+
+Optional
+[x] Visual reports             Installed     Enabled
+[ ] Excel support              Not installed [ Install ]
+[x] ML diagnostics             Installed     Enabled
+[ ] Deep models                Not installed [ Install ]
+[ ] Advanced text/NLP          Not installed [ Install ]
+[ ] Polars backend             Not installed [ Install ]
+[ ] Arrow backend              Not installed [ Install ]
+[ ] SQL adapters               Not installed [ Install ]
+[ ] Cloud filesystems          Not installed [ Install ]
+
+Space  Toggle enabled state
+Enter  Install / Open details
+R      Remove
+```
+
+### Installation flow
+
+Selecting `Install` should show exactly what will happen:
+
+```text
+Install: Deep Models
+
+Provides
+- Autoencoder anomaly detector
+- DeepSVDD anomaly detector
+- Optional temporal CNN/TCN diagnostics
+
+Python packages to install
+  framevitals[deep]
+
+Estimated package download: 220 MB
+Model downloads: none until a model is explicitly selected
+
+Command
+  /path/to/python -m pip install "framevitals[deep]"
+
+[ Install ] [ Cancel ]
+```
+
+Rules:
+- Never install implicitly during `import framevitals`.
+- Never install from a normal non-interactive analysis command.
+- Interactive install requires confirmation.
+- Show which Python environment will be modified.
+- Display installation errors instead of hiding them.
+- Allow user to copy the command and run it manually.
+
+## Model manager
+
+Python add-ons and model weights are different resources and should be managed separately.
+
+```text
+Models
+
+Semantic Type Classifier
+[ ] semantic-types-small       24 MB       [ Download ]
+
+Text
+[ ] text-embeddings-small      90 MB       [ Download ]
+
+Time Series
+[ ] temporal-anomaly-tiny      18 MB       [ Download ]
+
+Anomaly
+[ ] tabular-ae-reference       12 MB       [ Download ]
+```
+
+Model registry metadata should include:
+- model ID
+- model version
+- compatible FrameVitals versions
+- task
+- file size
+- checksum
+- license
+- source
+- framework/runtime requirement
+- expected hardware
+
+Downloaded weights should be cached in an OS-appropriate FrameVitals data directory, not in the working project directory.
+
+## Configuration screen
+
+Configuration should be editable interactively but stored in a human-readable file.
+
+Suggested conceptual config:
+
+```toml
+[analysis]
+preset = "standard"
+statistics = true
+relationships = true
+ml_readiness = true
+deep_models = false
+ai_interpretation = false
+
+[resources]
+workers = 4
+max_memory = "4GB"
+max_time_seconds = 120
+gpu = "auto"
+sampling = "adaptive"
+
+[reporting]
+terminal = true
+html = false
+json = false
+
+[privacy]
+pii_detection = true
+show_raw_sensitive_examples = false
+```
+
+## Configuration precedence
+
+Highest priority wins:
+
+1. Explicit Python arguments / command flags
+2. Project configuration (`framevitals.toml`)
+3. User configuration
+4. Environment variables
+5. Selected preset
+6. Built-in defaults
+
+`framevitals config explain` should show where each effective value came from.
+
+## Scriptable command design
+
+Interactive actions should have deterministic command equivalents.
+
+```bash
+# analysis
+framevitals analyze data.csv --preset deep
+framevitals analyze data.csv --target churn --html report.html --json report.json
+framevitals plan data.csv --preset exhaustive --explain
+
+# configuration
+framevitals config show
+framevitals config set analysis.deep_models true
+framevitals config profile use laptop
+framevitals config explain
+
+# add-ons
+framevitals addons list
+framevitals addons install deep
+framevitals addons enable deep
+framevitals addons disable deep
+framevitals addons remove deep
+
+# models
+framevitals models list
+framevitals models install semantic-types-small
+framevitals models remove semantic-types-small
+
+# environment diagnostics
+framevitals doctor
+framevitals doctor --json
+```
+
+## Exit codes
+
+Suggested stable policy:
+- `0` — command completed / validation passed
+- `1` — validation or configured quality gate failed
+- `2` — invalid arguments/configuration
+- `3` — input/data loading failure
+- `4` — optional capability unavailable
+- `5` — internal execution failure
+
+Exact values can change before stabilization, but the policy should become documented and tested before 1.0.
+
+## Non-interactive safety
+
+When stdin/stdout is not interactive:
+- never prompt
+- never install packages
+- never download models unless an explicit install command was invoked
+- emit deterministic output
+- respect `--json`/`--quiet`
+- use stable exit codes
+
+## Doctor screen
+
+`framevitals doctor` should answer common environment questions:
+
+```text
+FrameVitals 0.x
+Python        3.13.7
+Platform      macOS arm64
+Core          OK
+ML pack       Installed
+Deep pack     Missing
+Polars        Missing
+PyArrow       Installed
+GPU runtime   Not detected
+Cache         /Users/me/Library/Caches/framevitals
+Config        ~/.config/framevitals/config.toml
+
+No blocking problems detected.
+```
+
+## UX principle
+
+The TUI should hide accidental complexity, not analytical detail. Users should be able to make common decisions with a toggle or button while still being able to inspect:
+- exactly what analysis is being run
+- why it was selected
+- what it costs
+- which optional dependency/model it uses
+- what command/change an install button performs
diff --git a/docs/architecture-proposal/DECISIONS.md b/docs/architecture-proposal/DECISIONS.md
new file mode 100644
index 0000000..396b0a3
--- /dev/null
+++ b/docs/architecture-proposal/DECISIONS.md
@@ -0,0 +1,136 @@
+# FrameVitals Architecture Decisions
+
+This file records decisions that were explicitly accepted during architecture planning so future implementation does not repeatedly reopen the same product questions without new evidence.
+
+These are direction-setting principles, not a frozen public API contract. Current package behavior and stability guarantees are defined by the maintained code, tests, and release documentation.
+
+## D-001 — Exhaustive capability, selective execution
+
+**Status:** Accepted
+
+FrameVitals may aim for a very broad and eventually exhaustive catalog of tabular-data diagnostics. A normal analysis must not run everything blindly.
+
+Default execution is selected by:
+- data applicability
+- preset/configuration
+- installed and enabled capabilities
+- resource budget
+- analysis dependencies
+
+`exhaustive` means every applicable installed analysis permitted by the active resource policy.
+
+---
+
+## D-002 — Interactive terminal UI plus scriptable CLI
+
+**Status:** Accepted direction
+
+Running `framevitals` with no explicit subcommand may eventually open an interactive terminal application when that capability is installed.
+
+Explicit commands such as `framevitals analyze`, `compare`, and `validate` remain deterministic and suitable for scripts/CI.
+
+The TUI and CLI should share the same configuration and execution engine.
+
+---
+
+## D-003 — Optional features use install actions/toggles
+
+**Status:** Accepted direction
+
+Optional capabilities should be discoverable rather than forcing users to research dependency groups manually.
+
+Any interactive installation action must still show the exact command/environment being modified and request confirmation.
+
+Installed and enabled are separate states.
+
+---
+
+## D-004 — No implicit package/model downloads
+
+**Status:** Accepted
+
+FrameVitals never installs Python packages or downloads model weights during normal import or ordinary analysis.
+
+Downloads occur only through explicit add-on/model installation actions or equivalent dedicated commands.
+
+---
+
+## D-005 — Deep learning is optional and task-specific
+
+**Status:** Accepted
+
+ML/DL is used only where it adds analytical value, for example:
+- nonlinear multivariate anomaly detection
+- temporal CNN/TCN diagnostics for ordered time-series windows
+- optional semantic-type second opinion
+- embedding-based text drift
+- nonlinear learnability diagnostics
+
+CNNs are not applied to generic unordered tabular rows merely because they are available.
+
+Deterministic/statistical fallbacks remain first-class.
+
+---
+
+## D-006 — FrameVitals is not an AutoML platform
+
+**Status:** Accepted
+
+Baseline models and nonlinear diagnostic models can measure learnability, leakage, feature behavior, or explainability. The product does not optimize around training/deploying a production model fleet.
+
+The central question remains: **what is true about this dataset, what is wrong, what changed, and what should the user investigate next?**
+
+---
+
+## D-007 — Result/report UX precedes algorithm count
+
+**Status:** Accepted
+
+Stable result objects/schema, normalized findings, full JSON output, good terminal rendering, and useful reports are higher priority than continuously adding isolated algorithms.
+
+New analysis is only valuable when users can consume and trust the result.
+
+---
+
+## D-008 — One reusable analysis context / exact-once facts
+
+**Status:** Accepted direction
+
+Modules should consume reusable facts from the execution context rather than independently rescanning the same dataset for missingness, dtypes, semantic roles, samples, moments, correlations, and related facts.
+
+When the source pass has already established an exact sufficient statistic, downstream diagnostics should reuse it rather than replace it with a bounded-sample estimate.
+
+---
+
+## D-009 — Cleaning is plan-first
+
+**Status:** Accepted direction
+
+FrameVitals may recommend/simulate cleaning operations. It should not silently mutate user data.
+
+Preferred workflow:
+1. detect issue
+2. propose cleaning action
+3. estimate/simulate impact when possible
+4. user explicitly applies approved transformations
+5. record an audit trail
+
+---
+
+## D-010 — Public project website is separate from analysis runtime
+
+**Status:** Accepted
+
+The public website is primarily for product explanation, demos, installation, documentation, and benchmark/transparency information.
+
+It does not need to become a dataset-upload service. The website and generated reports can share a coherent visual language while remaining separate products.
+
+---
+
+## D-011 — Planning material is preserved but not release contract
+
+**Status:** Accepted
+
+Architecture proposal material is preserved under `docs/architecture-proposal/` in the integration history. It should not be treated as the current release contract.
+
+Implementation changes still land through normal development/review and must earn their place through correctness, resource, compatibility, and benchmark evidence.
diff --git a/docs/architecture-proposal/IMPLEMENTATION_ROADMAP.md b/docs/architecture-proposal/IMPLEMENTATION_ROADMAP.md
new file mode 100644
index 0000000..2e75725
--- /dev/null
+++ b/docs/architecture-proposal/IMPLEMENTATION_ROADMAP.md
@@ -0,0 +1,274 @@
+# FrameVitals Implementation Roadmap
+
+This document turns the future architecture proposal into a practical sequence of pull requests. It is a planning reference, not a promise that every item ships in the named release.
+
+## Operating rule
+
+FrameVitals should become **exhaustive in capability and selective in execution**. New capabilities should be added behind stable interfaces, configuration, applicability checks, and resource budgets rather than wired directly into one ever-growing pipeline.
+
+## Build order
+
+### PR 1 — Result model foundation
+**Priority:** P0
+
+Create a stable internal/public result layer without breaking the current dictionary-returning API.
+
+Proposed modules:
+- `src/framevitals/result.py`
+- `src/framevitals/findings.py`
+- `src/framevitals/metadata.py`
+
+Deliverables:
+- `AnalysisResult` or an internal result object with mapping compatibility.
+- Normalized `Finding` structure: code, title, severity, confidence, evidence, affected columns, recommendation, method.
+- `to_dict()` and JSON-safe serialization.
+- Existing `fv.analyze()` output remains backward compatible during migration.
+
+Acceptance criteria:
+- Existing tests still pass.
+- Old dictionary access remains available.
+- Result schema receives an explicit version.
+
+---
+
+### PR 2 — Full report export and terminal renderer
+**Priority:** P0
+
+Fix the current gap between the rich Python result and the summary-only CLI output.
+
+Proposed modules:
+- `src/framevitals/reporting/terminal.py`
+- `src/framevitals/reporting/json.py`
+
+Deliverables:
+- Human-friendly terminal summary.
+- Full JSON export from CLI.
+- Clear distinction between terminal summary and machine-readable full result.
+- Stable exit behavior.
+
+Acceptance criteria:
+- `framevitals analyze data.csv --output report.json` can intentionally write the full report.
+- Terminal output is concise and does not dump enormous nested JSON by default.
+
+---
+
+### PR 3 — Standalone HTML report
+**Priority:** P0
+
+Proposed modules:
+- `src/framevitals/reporting/html.py`
+- `src/framevitals/reporting/assets.py`
+
+Deliverables:
+- Self-contained HTML report requiring no server.
+- Overview, health, findings, missingness, column profiles, relationships, anomalies, target intelligence, drift when available, and recommendations.
+- Deterministic report generation from an existing result object.
+
+Acceptance criteria:
+- Report works offline in a browser.
+- No user dataset is uploaded anywhere.
+- HTML generation does not re-run analysis.
+
+---
+
+### PR 4 — Configuration and presets
+**Priority:** P0
+
+Proposed modules:
+- `src/framevitals/config.py`
+- `src/framevitals/config_presets.py`
+- `src/framevitals/config_profiles.py`
+
+Deliverables:
+- Typed configuration object.
+- Presets: `quick`, `standard`, `deep`, `exhaustive`, `custom`.
+- Resource policy: workers, memory budget, time budget, sampling policy, GPU preference.
+- Analysis-category toggles.
+- Config precedence rules.
+
+Suggested precedence:
+1. explicit Python/CLI arguments
+2. project config
+3. user config
+4. environment variables
+5. preset defaults
+
+Acceptance criteria:
+- Same config can be used from Python and CLI.
+- Config validation produces useful errors.
+
+---
+
+### PR 5 — AnalysisContext and reusable cache
+**Priority:** P0
+
+Proposed modules:
+- `src/framevitals/execution/context.py`
+- `src/framevitals/execution/cache.py`
+- `src/framevitals/execution/sampling.py`
+
+Deliverables:
+- One context per run containing loaded data/backend, shape, profile, roles, fingerprints, reusable samples, configuration, seed, installed capabilities, and cached intermediates.
+- Modules request reusable facts from the context rather than rescanning the dataset.
+
+Acceptance criteria:
+- Current outputs remain equivalent for deterministic analyses.
+- Benchmarks demonstrate fewer full-data scans on representative datasets.
+
+---
+
+### PR 6 — Planner-controlled execution
+**Priority:** P0
+
+Evolve `analysis_inventory.py` + `analysis_selector.py` into a real execution planner.
+
+Proposed modules:
+- `src/framevitals/planner.py`
+- `src/framevitals/execution/budget.py`
+
+Deliverables:
+- Registry entries declare inputs, applicability, cost, dependencies, outputs, optional pack, and resource class.
+- Planner produces an explainable execution plan.
+- `framevitals plan data.csv --explain` shows what will run and why.
+- Exhaustive mode means every applicable installed analysis that fits the configured policy.
+
+Acceptance criteria:
+- The planner actually controls execution rather than only describing it.
+- Skipped analyses record a reason.
+
+---
+
+### PR 7 — Interactive CLI/TUI foundation
+**Priority:** P1
+
+Proposed modules:
+- `src/framevitals/cli_tui.py`
+- `src/framevitals/doctor.py`
+
+Deliverables:
+- Running `framevitals` without a subcommand opens the interactive interface when the TUI is available.
+- Analyze, Compare, Validate, Reports, Configuration, Add-ons, Models, Doctor.
+- Traditional subcommands remain fully usable for scripts and CI.
+
+Acceptance criteria:
+- No TUI dependency is imported during a normal library import.
+- Non-interactive behavior remains stable.
+
+---
+
+### PR 8 — Add-on manager and capability registry
+**Priority:** P1
+
+Proposed modules:
+- `src/framevitals/addons.py`
+- `src/framevitals/capabilities.py`
+- `src/framevitals/models/registry.py`
+
+Deliverables:
+- Installed/enabled are separate states.
+- Discover whether optional dependencies are installed.
+- TUI install/remove/toggle actions.
+- Scriptable equivalents: `addons list/install/remove/enable/disable`.
+- Exact command displayed before modifying the active environment.
+
+Acceptance criteria:
+- Importing FrameVitals never installs anything.
+- Installation requires explicit confirmation in interactive mode.
+
+---
+
+### PR 9 — Semantic types and deeper profiling
+**Priority:** P1
+
+Proposed modules:
+- `src/framevitals/semantic_types.py`
+- `src/framevitals/quality/keys.py`
+- `src/framevitals/quality/missingness.py`
+
+Deliverables:
+- Stronger semantic types: ID, UUID, email, URL, IP, phone-like, currency, percentage, geospatial-like, date/time, free text, code/category.
+- Candidate primary keys and composite-key hints.
+- Missingness patterns and co-occurrence.
+- Categorical normalization suggestions.
+- Optional small character model can later act as a second opinion when rule confidence is low.
+
+---
+
+### PR 10 — Target intelligence and relationship engine
+**Priority:** P1
+
+Proposed modules:
+- `src/framevitals/target_intelligence.py`
+- `src/framevitals/relationships.py`
+- `src/framevitals/ml/label_quality.py`
+
+Deliverables:
+- Target/task inference.
+- Class imbalance and target quality.
+- Proxy, temporal, and split leakage.
+- Numeric↔numeric, categorical↔categorical, numeric↔categorical relationships.
+- Feature stability/redundancy signals.
+
+Optional model support:
+- nonlinear baseline as a diagnostic, not AutoML.
+
+---
+
+### PR 11 — Drift, snapshots, and monitoring foundation
+**Priority:** P1
+
+Proposed modules:
+- `src/framevitals/drift/`
+- `src/framevitals/monitoring/snapshot.py`
+- `src/framevitals/monitoring/store.py`
+
+Deliverables:
+- Pluggable drift methods and automatic method selection.
+- Drift severity and confidence/evidence.
+- Schema diff.
+- Result snapshots and local history.
+- Compare current result against a baseline snapshot.
+
+---
+
+### PR 12 — Scale/backends and first optional model pack
+**Priority:** P2
+
+Proposed modules:
+- `src/framevitals/backends/`
+- `src/framevitals/io/`
+- `src/framevitals/models/deep_anomaly.py`
+- `src/framevitals/models/runtime.py`
+
+Deliverables:
+- Polars/PyArrow/Narwhals strategy.
+- Parquet and chunked/streaming paths.
+- Large-data sampling/budget behavior.
+- Optional anomaly pack: autoencoder and/or DeepSVDD alongside classical detectors.
+- Optional GPU selection only when useful and installed.
+
+Acceptance criteria:
+- Core install stays usable without deep-learning frameworks.
+- Model pack failure never removes deterministic fallback analysis.
+
+## Parallel workstreams
+
+The public project website can be developed in parallel because it should initially use curated/precomputed demo data rather than depend on the full backend roadmap.
+
+Recommended parallel streams:
+- **Core:** PR 1–6
+- **Experience:** PR 2–3, 7–8
+- **Analysis depth:** PR 9–11
+- **Scale/models:** PR 12+
+- **Website:** independent visual/demo frontend with shared report design language
+
+## Release philosophy
+
+Do not ship a release because the project gained a certain number of algorithms. Ship when a coherent user workflow becomes materially better.
+
+Examples:
+- result/report release
+- configuration/planner release
+- interactive/add-on release
+- monitoring release
+- scale/backend release
diff --git a/docs/architecture-proposal/PROPOSED_MODULE_TREE.md b/docs/architecture-proposal/PROPOSED_MODULE_TREE.md
new file mode 100644
index 0000000..0559b74
--- /dev/null
+++ b/docs/architecture-proposal/PROPOSED_MODULE_TREE.md
@@ -0,0 +1,256 @@
+# Proposed Future Module Tree
+
+This is a destination architecture, **not** a request to rename/move every current module immediately. Existing public imports should remain compatible while internal organization evolves gradually.
+
+```text
+src/framevitals/
+├── __init__.py
+├── api.py
+├── result.py
+├── findings.py
+├── metadata.py
+├── errors.py
+├── config.py
+├── config_presets.py
+├── config_profiles.py
+├── capabilities.py
+├── addons.py
+├── doctor.py
+│
+├── execution/
+│   ├── __init__.py
+│   ├── context.py
+│   ├── planner.py
+│   ├── inventory.py
+│   ├── budget.py
+│   ├── cache.py
+│   ├── sampling.py
+│   └── scheduler.py
+│
+├── io/
+│   ├── __init__.py
+│   ├── loader.py
+│   ├── csv.py
+│   ├── excel.py
+│   ├── json.py
+│   ├── parquet.py
+│   ├── sql.py
+│   └── cloud.py
+│
+├── backends/
+│   ├── __init__.py
+│   ├── base.py
+│   ├── pandas.py
+│   ├── polars.py
+│   ├── arrow.py
+│   └── narwhals_adapter.py
+│
+├── profiling/
+│   ├── __init__.py
+│   ├── structural.py
+│   ├── semantic_types.py
+│   ├── column_roles.py
+│   ├── dataset_signals.py
+│   ├── keys.py
+│   ├── missingness.py
+│   └── freshness.py
+│
+├── quality/
+│   ├── __init__.py
+│   ├── health.py
+│   ├── duplicates.py
+│   ├── consistency.py
+│   ├── ranges.py
+│   ├── categories.py
+│   └── pii.py
+│
+├── contracts/
+│   ├── __init__.py
+│   ├── infer.py
+│   ├── validate.py
+│   ├── rules.py
+│   ├── policy.py
+│   ├── diff.py
+│   └── schema.py
+│
+├── statistics/
+│   ├── __init__.py
+│   ├── descriptive.py
+│   ├── distributions.py
+│   ├── robust.py
+│   ├── testing.py
+│   ├── relationships.py
+│   ├── multicollinearity.py
+│   └── segments.py
+│
+├── anomaly/
+│   ├── __init__.py
+│   ├── ensemble.py
+│   ├── univariate.py
+│   ├── classical.py
+│   └── scoring.py
+│
+├── drift/
+│   ├── __init__.py
+│   ├── compare.py
+│   ├── numeric.py
+│   ├── categorical.py
+│   ├── text.py
+│   ├── schema.py
+│   ├── methods.py
+│   └── selection.py
+│
+├── target/
+│   ├── __init__.py
+│   ├── intelligence.py
+│   ├── task.py
+│   ├── leakage.py
+│   ├── imbalance.py
+│   ├── label_quality.py
+│   └── candidates.py
+│
+├── ml/
+│   ├── __init__.py
+│   ├── readiness.py
+│   ├── preprocessing.py
+│   ├── baseline.py
+│   ├── leaderboard.py
+│   ├── diagnostics.py
+│   ├── feature_importance.py
+│   └── explainability.py
+│
+├── time_series/
+│   ├── __init__.py
+│   ├── detection.py
+│   ├── quality.py
+│   ├── diagnostics.py
+│   ├── seasonality.py
+│   ├── change_points.py
+│   └── anomaly.py
+│
+├── text/
+│   ├── __init__.py
+│   ├── profile.py
+│   ├── quality.py
+│   ├── duplication.py
+│   ├── pii.py
+│   └── drift.py
+│
+├── cleaning/
+│   ├── __init__.py
+│   ├── plan.py
+│   ├── suggestions.py
+│   ├── simulate.py
+│   ├── apply.py
+│   └── audit.py
+│
+├── monitoring/
+│   ├── __init__.py
+│   ├── snapshot.py
+│   ├── store.py
+│   ├── history.py
+│   └── policy.py
+│
+├── models/
+│   ├── __init__.py
+│   ├── registry.py
+│   ├── manifest.py
+│   ├── runtime.py
+│   ├── semantic_types.py
+│   ├── deep_anomaly.py
+│   ├── temporal.py
+│   └── embeddings.py
+│
+├── reporting/
+│   ├── __init__.py
+│   ├── terminal.py
+│   ├── json.py
+│   ├── html.py
+│   ├── pdf.py
+│   ├── notebook.py
+│   ├── charts.py
+│   ├── compare_html.py
+│   └── privacy.py
+│
+├── plugins/
+│   ├── __init__.py
+│   ├── registry.py
+│   ├── checks.py
+│   ├── analyzers.py
+│   └── hooks.py
+│
+├── ai/
+│   ├── __init__.py
+│   ├── insights.py
+│   ├── agent.py
+│   ├── tools.py
+│   └── brief.py
+│
+├── cli.py
+└── cli_tui.py
+```
+
+## Migration philosophy
+
+Do **not** perform a giant move-only refactor immediately. Prefer:
+
+1. introduce a new namespace when implementing a new coherent capability
+2. move an old module only when it is actively being changed or causes architectural friction
+3. keep compatibility shims for public imports where necessary
+4. delete shims only through documented deprecation cycles
+
+## Mapping from important current modules
+
+| Current module | Future home/direction |
+|---|---|
+| `pipeline.py` | gradually replaced/orchestrated by `execution/context.py` + `execution/planner.py` |
+| `analysis_inventory.py` | `execution/inventory.py` |
+| `analysis_selector.py` | `execution/planner.py` |
+| `profiler.py` | `profiling/structural.py` |
+| `column_roles.py` | `profiling/column_roles.py` + semantic typing layer |
+| `dataset_signals.py` | `profiling/dataset_signals.py` |
+| `health_score.py` | `quality/health.py` |
+| `contracts.py` | split gradually into `contracts/` package |
+| `drift_analysis.py` | split gradually into `drift/` package |
+| `anomaly_ensemble.py` | `anomaly/ensemble.py` |
+| `ml_readiness.py` | `ml/readiness.py` |
+| `target_analyzer.py` | `target/intelligence.py` |
+| `target_leakage.py` | `target/leakage.py` |
+| `time_series.py` | split into `time_series/` package |
+| `text_profile.py` | `text/profile.py` |
+| `cleaner.py` | replaced by plan/simulate/apply/audit workflow in `cleaning/` |
+| `visualizer.py` | `reporting/charts.py` |
+| PDF/report modules | `reporting/` |
+| `ai_agent.py`, `ai_insights.py`, `agent_tools.py`, `agent_brief.py` | isolated under optional `ai/` namespace |
+
+## Dependency direction
+
+A useful dependency rule is:
+
+```text
+api/result/reporting
+        ↓
+execution planner/context
+        ↓
+analysis domains
+        ↓
+profiling/backends/config
+```
+
+Analysis-domain modules should **not** import the CLI/TUI or website code. Reporting should consume result structures rather than cause analyses to rerun. Optional AI should consume structured findings/results rather than become a required dependency of deterministic modules.
+
+## Plugin boundary
+
+Long term, third-party analysis plugins should register through a stable plugin API instead of monkey-patching the pipeline.
+
+A plugin should declare:
+- ID/name/version
+- supported FrameVitals plugin API version
+- applicability predicate
+- required inputs/context facts
+- resource class
+- optional dependency requirements
+- output/finding schema
+- whether failure is fatal or optional
+
+This allows FrameVitals to be exhaustive without requiring every niche analysis to live in the core repository forever.
diff --git a/docs/architecture-proposal/README.md b/docs/architecture-proposal/README.md
new file mode 100644
index 0000000..89fb461
--- /dev/null
+++ b/docs/architecture-proposal/README.md
@@ -0,0 +1,24 @@
+# FrameVitals Architecture Proposal Archive
+
+These documents were transferred from the former `docs/architecture-proposal` working branch into `develop/august` so the useful planning work can be preserved before branch cleanup.
+
+They are **planning references, not the current public API or release contract**. Several originally proposed items have already evolved into implemented FrameVitals features, including result objects, source-aware execution, execution budgets, streaming/Arrow support, native Rust kernels, planning APIs, monitoring snapshots, quality gates, and capability-oriented optional dependencies.
+
+For current behavior, prefer the maintained documentation in the parent `docs/` directory and the package code/tests.
+
+## Preserved planning documents
+
+- `IMPLEMENTATION_ROADMAP.md` — original staged implementation sequence and product philosophy.
+- `BENCHMARK_ACCEPTANCE_PLAN.md` — correctness/performance/memory/backend acceptance ideas.
+- `CAPABILITY_PACKS.md` — optional capability-pack and dependency-management design.
+- `CLI_TUI_SPEC.md` — future interactive terminal design ideas.
+- `DECISIONS.md` — accepted long-term product/architecture principles.
+- `PROPOSED_MODULE_TREE.md` — proposed modular organization for future growth.
+
+Generated PDF/LaTeX artifacts and the proposal-only build workflow were intentionally **not** transferred. Source planning content belongs in Git; generated planning artifacts do not need to remain on an active development branch.
+
+## Core rule retained from the proposal
+
+> FrameVitals should be exhaustive in capability, not exhaustive in what it runs by default.
+
+That principle remains compatible with the current direction: the execution engine should choose the cheapest statistically defensible work based on applicability, scale, installed capabilities, backend availability, and requested analysis depth.

From 378a60f2454b827def4f9f6a84fc5601eb3391c2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 12:26:44 +0530
Subject: [PATCH 425/439] chore: prepare FrameVitals 0.2.0 release candidate

---
 CHANGELOG.md                     | 205 ++++++++++++-------------------
 RELEASING.md                     | 134 ++++++++++----------
 pyproject.toml                   |   2 +-
 rust/framevitals-core/Cargo.toml |   2 +-
 rust/framevitals-py/Cargo.toml   |   2 +-
 src/framevitals/__init__.py      |  49 ++------
 6 files changed, 156 insertions(+), 238 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4bcdb3b..d22c936 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,104 +2,83 @@
 
 All notable user-facing changes to FrameVitals are documented here.
 
-FrameVitals follows semantic versioning while the public API matures. The 0.x series may still include breaking changes, which will be called out in release notes.
+FrameVitals follows semantic versioning while the public API matures. The 0.x series may still include breaking changes, which are called out in release notes.
 
 ## Unreleased
 
-Development continues on the development branches. Changes intended for the next release should be documented here before they are promoted to `main`.
+No user-facing changes are currently queued beyond 0.2.0.
 
-### Added
+## 0.2.0 - 2026-08-17
+
+FrameVitals 0.2.0 is the first release built around the source-aware Arrow/Rust execution architecture rather than treating every dataset as a pandas-first workload.
+
+### Highlights
+
+- Added bounded Arrow streaming for large Parquet, CSV/TSV, PyArrow, Arrow C Stream, Polars-through-Arrow, and optional DuckDB relation inputs.
+- Added a native Rust execution backend with direct Arrow `RecordBatch` profiling, mergeable numeric state, native categorical sketches, and full-stream log-quantile sketches.
+- Added exact-once reuse so downstream Deep/Research diagnostics consume already-known full-stream facts instead of recomputing weaker bounded-sample estimates.
+- Added exact full-stream count, missingness, mean, variance/std, min/max, skewness, and excess kurtosis through mergeable central moments up to M4.
+- Replaced fixed evenly spaced row sampling with deterministic stratified-jitter sampling to reduce periodic/structured-data aliasing while preserving reproducibility and row order.
+- Added explicit execution provenance for full-stream, projected-column, sketch, bounded-sample, and materialized operations.
+- Added physical large-scale benchmark coverage up to 500,000 × 10,000 (5 billion logical cells) with native/fallback routing, accuracy, exact-once, and no-full-materialization checks.
+- Added mixed statistical ground-truth validation across native and fallback execution.
+- PyPI publishing now builds native ABI3 wheels for common Linux/macOS/Windows targets while retaining a portable pure-Python fallback wheel and source distribution.
+
+### Analysis and public APIs
+
+- Added focused APIs for `profile`, `roles`, `health`, `ml_readiness`, `quality`, `statistics`, `anomalies`, `relationships`, and `target_analysis` so callers do not need to run unrelated pipeline stages.
+- Added `framevitals.plan()` for previewing execution budgets, source capabilities, applicable work, and bounded planning behavior.
+- Added `framevitals.inspect_source()` and the matching CLI command for source metadata/capability inspection without analysis.
+- Added data-contract inference with `framevitals.infer_contract()` and structured validation with `framevitals.validate()`.
+- Added `framevitals.gate()` plus a reusable GitHub Action for CI-friendly contract, drift, and custom-check verdicts.
+- Added `framevitals.check()`, `run_checks()`, and opt-in third-party check discovery through Python entry points.
+- Added compact versioned analysis snapshots, snapshot history, and snapshot-to-snapshot monitoring diffs.
+- Added structured dict-compatible result objects including analysis, diagnostic, drift, validation, check, and gate results.
+- Added `system-info`, snapshot, monitoring, planning, inspection, validation, and gate CLI workflows.
+
+### Execution architecture
+
+- Full `framevitals.analyze()` now resolves generic dataset sources and uses bounded source-aware execution when the source supports streaming/projection.
+- Ultra-wide sources use deterministic schema projection under explicit cell/column budgets instead of scanning every source cell blindly.
+- Streaming analysis performs one authoritative full-source profile scan and schedules only genuinely row-dependent modules on the bounded working sample.
+- Numeric native execution accepts Arrow record batches directly through the Arrow C Data/PyCapsule interface, avoiding the former Arrow → NumPy float64 → per-column Python bridge on supported types.
+- Streaming numeric profiling uses a specialized moments + log-quantile state instead of paying for unrelated HLL/heavy-hitter/reservoir work on every numeric cell.
+- Parquet sources cache metadata/schema/file handles per source to avoid repeated metadata parsing.
+- Research inference uses adaptive statistically defensible methods rather than blindly running expensive resampling on large bounded samples.
+- Execution budgets now bound expensive quality, deep-statistics, anomaly, time-series, relationship, bootstrap, and distribution work by source scale and analysis mode.
+
+### Correctness and statistical fidelity
+
+- Full-stream missing counts, numeric counts, min/max, and moments are reused downstream with explicit provenance.
+- Deep/Research shape summaries can reuse exact full-stream skewness/kurtosis while sample-dependent distribution fits, tests, intervals, and relationship diagnostics remain explicitly sample-scoped.
+- Streaming ML-readiness, health, roles, quality, relationships, statistics, and anomaly results now distinguish exact/full-stream facts from bounded estimates.
+- Categorical native profiling reports full-stream approximate cardinality/heavy-hitter provenance rather than pretending native sketches were row samples.
+- Anderson-Darling normality diagnostics support modern and legacy SciPy contracts without relying on a fixed critical-value index.
+- Large-source sampling regression tests cover periodic aliasing and native/fallback provenance contracts.
+
+### Performance and scale validation
+
+FrameVitals benchmarks are not advertised as universal speedups. Results depend on mode, backend, source shape, storage, and statistical fidelity.
+
+Validated physical workloads include 1B, 2.5B, and 5B logical cells. On the 5B 500k × 10k workload, the native full-stream profiling kernel remains competitive while retaining full-stream native quantile sketches where the fallback may choose bounded row-sample quantiles for cost. All published stress workflows enforce no full materialization and correctness tolerances.
+
+### Packaging and compatibility
 
-- Data-contract inference through `framevitals.infer_contract()` with versioned contract schemas, tolerant null-rate and numeric-bound expectations, categorical domains, optional-column semantics, and uniqueness hints
-- Structured exact contract validation through `framevitals.validate()`
-- `framevitals.gate()` for one CI-friendly pass/warn/fail verdict combining contract validation, drift, and optional custom checks
-- `framevitals gate` CLI command with configurable drift warning/failure thresholds, optional contract enforcement, JSON output, and CI exit codes
-- Reusable root `action.yml` GitHub Action for running FrameVitals quality gates with status/result outputs
-- Path-scoped smoke CI for the reusable Gate Action
-- `framevitals plan` for previewing applicable modules, execution budgets, source capabilities, and bounded planning samples
-- `framevitals.inspect_source()` for inspecting source shape, storage kind, format, materialization state, projection support, and streaming capability without running diagnostics
-- `framevitals inspect` CLI command with terminal/JSON output and optional persisted source metadata
-- `framevitals snapshot` and `framevitals compare-snapshots` CLI commands for compact monitoring state, snapshot diffs, JSON persistence, and CI-friendly change exit codes
-- `framevitals system-info` CLI command for reproducible CPU/native/GPU capability reports in support and performance workflows
-- Compact versioned analysis snapshots with deterministic fingerprints, JSON persistence, and snapshot-to-snapshot schema/health/ML-readiness/finding diffs
-- `SnapshotHistory` for persistent compact monitoring timelines, latest/previous lookup, labels, and recent-state comparison without storing raw datasets
-- Dict-compatible result objects including `AnalysisResult`, `DiagnosticResult`, `DriftResult`, `ValidationResult`, `CheckResult`, `GateResult`, and per-column result views
-- `framevitals.check()` and `framevitals.run_checks()` for exact user-defined DataFrame invariants with warning/error severities and structured findings
-- Opt-in `framevitals.discover_checks()` plugin discovery through the `framevitals.checks` Python entry-point group
-- Installable third-party check-plugin example plus CI that installs, discovers, executes, and gates on external entry points
-- Focused public APIs for profiling, column roles, health, ML readiness, quality, statistics, anomalies, relationships, and target analysis without running unrelated pipeline stages
-- Source abstraction with metadata inspection, projection, and streaming capability discovery
-- Bounded Arrow-backed Parquet execution for public profiling, health, ML-readiness, roles, quality, statistics, anomaly, relationship, target-aware, drift, gate, planning, and full-analysis paths where those diagnostics support bounded execution
-- Optional Arrow-backed CSV and TSV streaming with exact row-count metadata, projected record batches, and pandas fallback for incompatible inputs
-- Native PyArrow `Table` and `RecordBatch` inputs through the same source-aware streaming engine
-- Arrow C Stream / PyCapsule-compatible table input support without requiring a library-specific adapter for every producer
-- Tested Polars DataFrame interoperability through the standard Arrow C Stream boundary without adding Polars as a FrameVitals runtime dependency
-- Optional lazy DuckDB relation source with exact count/schema metadata, projection pushdown, Arrow record batches, and full pandas materialization only for exact/full-row APIs
-- Optional `duckdb` dependency group containing DuckDB plus its Arrow transport
-- Dedicated Arrow/DuckDB interoperability CI coverage
-- Versioned execution-provenance schema v1 with shared `method`, `full_materialization`, source, sampling, strategy, component, and reason fields while preserving legacy operation-specific metadata during the `0.x` migration
-- Optional Rust native core and Python bridge for accelerated streaming numeric state, cardinality/quantile sketches, and backend routing
-- Dedicated Arrow/NumPy-fallback and Rust/native CI coverage
-- Reproducible performance benchmark workflow with time/RSS summaries and retained JSON artifacts
-- Catastrophic performance guardrails for deterministic core and Parquet profiling scenarios with retained measurement artifacts
-- Minimum-dependency CI that installs and tests the declared core dependency floors separately from optional extras
-- Windows and macOS public-API smoke CI alongside the full Linux matrix
-- CodeQL scanning for Python and JavaScript/TypeScript on integration/release branches, pull requests, manual runs, and a scheduled cadence
-- PEP 561 `py.typed` marker for downstream type-checking support
-- Optional `excel` dependency group for XLS/XLSX readers
-- Optional `plot` dependency group for Matplotlib/Seaborn chart and report plotting support
-- Optional `docs` dependency group plus a MkDocs Material documentation site covering source-aware execution, execution provenance, result objects, monitoring, quality gates, custom checks, extension authors, and stability guarantees
-- Strict path-scoped documentation CI using `mkdocs build --strict`
-- Pre-commit and pre-push contributor guardrails mirrored by dedicated CI
-- `SUPPORT.md` guidance for reproducible issue reports without requiring users to share sensitive datasets
-- Stability/compatibility documentation defining the intended `0.x` public-surface, result-schema, execution-provenance, optional-dependency, Python-version, and platform expectations
-- Contributor architecture guidance covering canonical layers, streaming provenance, optional-dependency boundaries, and performance-sensitive changes
-- Focused examples for domain-specific quality gates and persistent snapshot monitoring
-
-### Changed
-
-- Full `framevitals.analyze()` now resolves generic `DatasetSource` inputs and dispatches streaming-capable sources through a bounded source-aware pipeline when artifacts do not require materialization
-- Drift comparison now preserves exact source shape metadata while bounding value-distribution work for streaming sources
-- Flask `/api/compare` now routes through the same canonical source-aware drift engine as Python and CLI callers
-- `framevitals.gate()` reuses the same canonical drift, validation, and custom-check engines instead of maintaining separate data-loading logic
-- Contract validation explicitly remains exact for uniqueness, allowed-value, nullability, and bound constraints rather than silently weakening them to sampled checks
-- Arbitrary custom Python checks explicitly remain exact and materialize non-pandas sources rather than silently sampling user-defined invariants
-- Top-level gate execution now reports whether any selected check family required full materialization
-- Materialization provenance is now based on actual execution semantics rather than assuming only file-backed sources can materialize
-- Focused statistics, anomaly, relationship, role, health, ML-readiness, quality, drift, validation, custom-check, and gate paths are converging on execution-provenance schema v1 without removing legacy metadata keys
-- Focused diagnostic APIs return dict-compatible `DiagnosticResult` objects with JSON/summary/source/execution ergonomics while preserving existing mapping payloads
-- The package-root `__all__` surface is now regression-tested as an exact `0.2` stabilization contract so accidental additions and removals both fail CI
-- `framevitals.api` is now a lazy compatibility facade over the canonical focused, analysis, planning, operations, check, plugin, and source-inspection engines instead of a second eager implementation, with focused return annotations aligned to `DiagnosticResult`
-- The installed CLI now routes directly through canonical APIs and no longer relies on a runtime monkeypatch shim
-- CLI `--version` works from a no-dependencies wheel smoke install without importing the analytics stack
-- The `all` extra now includes Arrow and DuckDB interoperability capabilities
-- The `dev` extra now includes the pre-commit tool required by the documented and CI-enforced contributor workflow
-- Excel engines moved out of the default dependency set into `framevitals[excel]`; AI-only Pydantic moved into `framevitals[ai]`
-- Matplotlib and Seaborn moved out of the default dependency set into `framevitals[plot]`
-- Explainability no longer requires Matplotlib at module import time; structured importance can run without plotting and SHAP chart rendering is optional
-- Flask startup no longer eagerly imports the agentic AI or report/plot stacks; those capabilities load only when their endpoints are used
-- Excel loading now reports a FrameVitals-specific install hint when the optional reader capability is missing
-- README and roadmap now describe the source-aware analyze → compare → validate → gate → monitor direction, source interoperability, exact-vs-bounded execution, optional capability groups, plugins, and the reusable Gate Action
-- Project metadata now advertises the package as typed and points documentation users at the maintained `docs/` tree
-- Package CI now requires the `py.typed` marker to be present in built wheels
-- Test and package workflows now use read-only default permissions and cancel stale runs on the same branch
-- Release documentation now separates stabilization-branch promotion to `dev` from intentional `dev` → `main` versioning and PyPI publication
-- Release publishing now verifies that the GitHub release tag, `pyproject.toml` version, and `framevitals.__version__` match before building/publishing
+- Added Python 3.11, 3.12, and 3.13 core coverage plus Windows/macOS smoke testing.
+- Added dedicated Arrow fallback, Rust/native, minimum-dependency, optional-feature, frontend, package-quality, documentation, and performance CI lanes.
+- Added PEP 561 `py.typed` packaging support.
+- Moved Excel, plotting, AI, Arrow, DuckDB, and other heavier capabilities behind explicit optional dependency groups where appropriate.
+- Release CI verifies the GitHub tag, Python package version, `pyproject.toml`, Rust core version, and Rust bridge version before publication.
+- Package CI verifies both fallback/native wheel contents and proves that pip prefers a compatible native wheel when both are available.
 
 ### Fixed
 
-- CSV/TSV no-Arrow fallback no longer relies on zero-argument `super()` in a slotted dataclass subclass, restoring Python 3.11/3.12 compatibility
-- Streaming ML-readiness results now disclose when duplicate rate is estimated from a bounded sample instead of reporting sampled data as exact
-- Streaming relationship metadata reports true source size instead of sample size
-- Streaming column-role cardinality metadata distinguishes bounded-sample cardinality from full-stream native estimates
-- CSV/TSV streaming sources preserve `FileSource` compatibility for existing callers and tests
-- Dedicated Arrow CI now exercises streaming statistics, drift, CSV/TSV sources, in-memory Arrow inputs, and the quality-gate CLI
-- Arrow `string_view` and `binary_view` inputs are normalized to compute-compatible Arrow types at the in-memory source boundary, fixing Polars sampling on PyArrow 25 without falling back to pandas
-- Optional AI and plotting tests now skip cleanly in core-only/minimum-dependency environments rather than requiring undeclared optional extras during test collection
-- Web runtime dependency boundaries no longer force AI/report/plot packages to import during Flask startup
-- Reusable Gate Action setup no longer relies on an invalid absolute `setup-python` cache dependency path
-- Exact validation/custom-check provenance now reports full materialization for Arrow and relation-backed inputs when they are converted to complete pandas DataFrames
-- Monitoring snapshot and system-capability CLI commands are now implemented by the installed parser/dispatcher rather than existing only as tests/documentation
+- Fixed periodic sampling aliasing caused by fixed evenly spaced global-row samples.
+- Fixed projected streaming missingness/quality semantics so denominators describe the profiled projection rather than silently using the true source width.
+- Fixed duplicate core recomputation in the streaming pipeline.
+- Fixed stale sampling/native-categorical provenance assertions and public execution labels.
+- Fixed SciPy Anderson-Darling compatibility and a pandas string-dtype selection deprecation.
+- Fixed several optional-dependency/import boundaries so core/lightweight usage does not eagerly require plotting, AI, web, or Excel stacks.
 
 ## 0.1.0 - 2026-08-15
 
@@ -107,40 +86,12 @@ First public alpha release of FrameVitals.
 
 ### Added
 
-- Installable `framevitals` package under `src/framevitals/`
-- Public `framevitals.analyze()` API for pandas DataFrames and supported dataset files
-- Public `framevitals.compare()` API for reference-vs-current drift analysis
-- `framevitals` command-line interface with analyze and compare commands
-- Data-health and ML-readiness scoring
-- Structural profiling and semantic column-role inference
-- Missingness, duplicate, cardinality, statistical, anomaly, target-aware, time-series, and text diagnostics
-- Drift comparison using PSI plus numeric/categorical statistical tests
-- Optional artifact generation for reusable Python workflows
-- Optional ML, AI, and Flask/React web dependency groups
-- Python 3.11, 3.12, and 3.13 test matrix
-- Package-build, wheel-boundary, Twine, and clean-install validation
-- PyPI Trusted Publishing release workflow
-- Dependabot configuration for Python, npm, and GitHub Actions
-- Open-source contributor, security, conduct, issue, and pull-request guidance
-
-### Changed
-
-- Project identity and package documentation moved from DataLens AI to FrameVitals
-- Reusable dashboard/report helpers moved into the canonical package
-- Top-level package import and CLI version path load the analytics pipeline lazily
-- Heavy ML, Ollama, and web dependencies moved into optional extras
-- Reusable Python analysis no longer writes cleaned datasets or charts unless `artifacts=True`
-- Repository layout is package-first, with the Flask API and React dashboard kept as optional interfaces
-- Frontend module documentation now points directly at canonical `src/framevitals/` implementations
-- Development setup standardized on `.venv`
-
-### Removed
-
-- Unused runtime dependencies including Optuna, imbalanced-learn, Pingouin, Plotly, Missingno, fpdf2, Jinja2, Joblib, and Loguru
-- Academic report, presentation, and whitepaper artifacts
-- Generated TypeScript compiler metadata and generated Vite JavaScript/config declarations
-- Tracked runtime-output directories for uploads, reports, and cleaned datasets
-- Large legacy demo CSVs and their one-off inspection harness
-- Redundant Streamlit console and its configuration
-- Legacy shell launcher/install scripts and duplicate `requirements.txt` development wrapper
-- Deprecated top-level `modules/` compatibility namespace after application imports moved to `framevitals.*`
+- Installable `framevitals` package under `src/framevitals/`.
+- Public `framevitals.analyze()` API for pandas DataFrames and supported dataset files.
+- Public `framevitals.compare()` API for reference-vs-current drift analysis.
+- `framevitals` command-line interface with analyze and compare commands.
+- Data-health and ML-readiness scoring.
+- Structural profiling and semantic column-role inference.
+- Missingness, duplicate, cardinality, statistical, anomaly, target-aware, time-series, and text diagnostics.
+- Drift comparison using PSI plus numeric/categorical statistical tests.
+- Optional artifact generation and ML/AI/web dependency groups.
diff --git a/RELEASING.md b/RELEASING.md
index 31aa643..6505253 100644
--- a/RELEASING.md
+++ b/RELEASING.md
@@ -1,94 +1,94 @@
 # Releasing FrameVitals
 
-FrameVitals is configured to publish to PyPI through GitHub Actions Trusted Publishing. No long-lived PyPI API token is required.
+FrameVitals publishes to PyPI through GitHub Actions Trusted Publishing. No long-lived PyPI API token is required.
 
 ## Branch model
 
-- `main` is the release-ready branch. Published releases and version tags come from `main`.
-- `dev` is the integration branch for ongoing development.
-- Feature and dependency-update pull requests should target `dev`.
-- Large stabilization branches should be promoted to `dev` only after their public contracts and specialized CI lanes are clean.
-- When a release is ready, promote the tested release changes from `dev` to `main`, update the version/changelog, and publish from `main`.
-
-## Stabilization branch → `dev`
-
-Promoting a development branch such as `develop/august` to `dev` is **not** a PyPI release and does not require changing the package version.
-
-Before opening or merging the promotion pull request:
-
-1. Confirm the branch is not behind `dev`, or deliberately reconcile any new `dev` commits.
-2. Confirm the package-root public surface test passes and any intended API change is documented.
-3. Confirm execution/result schema changes are additive or carry the appropriate schema-version change.
-4. Confirm `CHANGELOG.md` records user-facing changes under `Unreleased`.
-5. Confirm the core Tests and Package workflows are green.
-6. Confirm the specialized compatibility lanes relevant to the branch are green:
-   - documentation strict build;
-   - developer/pre-commit guardrails;
-   - declared minimum dependencies;
-   - Windows/macOS platform smoke;
-   - Arrow/DuckDB interoperability;
-   - Polars-through-Arrow interoperability;
-   - external check-plugin installation/discovery;
-   - native Rust bridge, when touched;
-   - reusable Gate Action smoke tests, when touched;
-   - performance guardrails for performance-sensitive changes.
-7. Confirm no known security-analysis failure is being carried into `dev`; CodeQL runs on `dev`, `main`, and their pull requests.
-8. Review the branch diff for generated files, temporary artifacts, stale compatibility shims, and accidental package-boundary changes.
-
-A large promotion pull request should summarize the public API, compatibility impact, execution-semantics changes, optional dependencies, performance implications, and remaining known limitations.
+- `main` is release-ready; version tags and published GitHub releases come from `main`.
+- `dev` is the normal integration branch.
+- Stabilization branches such as `develop/august` are promoted into `dev` only after their contracts and specialized CI lanes are clean.
+- A stabilization-branch merge is not itself a PyPI release.
 
-## One-time PyPI setup
+## Release candidate checklist
+
+Before promoting a release candidate toward `main`:
+
+1. Core Tests, Package, minimum-dependency, platform, Arrow/fallback, native Rust, optional-feature, frontend/package-quality, docs, and security-relevant CI must be green.
+2. Performance-sensitive changes must have benchmark evidence and accuracy/safety checks; do not publish unsupported universal speed claims.
+3. Review generated files, temporary artifacts, stale compatibility shims, and branch-only experiments before promotion.
+4. Update `CHANGELOG.md` with the release section and any known 0.x compatibility changes.
+5. Keep all release-version sources synchronized:
+   - GitHub release tag, e.g. `v0.2.0`;
+   - `pyproject.toml` → `[project].version`;
+   - `src/framevitals/__init__.py` → `__version__`;
+   - `rust/framevitals-core/Cargo.toml` → `[package].version`;
+   - `rust/framevitals-py/Cargo.toml` → `[package].version`.
+6. Confirm the package workflow builds and validates both the portable fallback wheel and a compatible native wheel.
+7. Confirm the clean-environment package smoke proves pip prefers the native wheel when both compatible native and universal fallback wheels are present.
+
+## Distribution model
+
+FrameVitals intentionally publishes two kinds of Python wheels:
+
+- **Native ABI3 wheels** for supported/common platforms. These include `framevitals._native` and are preferred automatically by pip when compatible.
+- **Portable fallback wheel** (`py3-none-any`) for environments without a published native wheel.
+
+The release workflow also publishes one source distribution.
+
+Current native release targets are:
 
-Before the first release:
+- Linux x86_64;
+- macOS arm64;
+- macOS x86_64;
+- Windows x86_64.
 
-1. Create or sign in to your PyPI account.
-2. Open your PyPI account's **Publishing** page and configure a pending Trusted Publisher for project name `framevitals`.
+The native extension uses a Python 3.11 ABI3 floor so one compatible wheel can serve supported Python 3.11+ versions on the same platform.
+
+## One-time PyPI setup
+
+1. Create/sign in to the PyPI account.
+2. Configure a pending Trusted Publisher for project `framevitals`.
 3. GitHub owner: `parthdongre`.
 4. Repository: `FrameVitals`.
 5. Workflow filename: `release.yml`.
 6. GitHub environment: `pypi`.
-7. In GitHub repository settings, create a `pypi` environment. Requiring manual approval for production publishing is recommended.
-
-A pending publisher does not reserve the package name until the first successful publication. If the repository is renamed later, update the PyPI Trusted Publisher configuration before publishing again.
+7. Create the matching `pypi` environment in GitHub repository settings; production approval is recommended.
 
-## `dev` → `main` release checklist
-
-Before promoting a release candidate from `dev` to `main`:
-
-1. Confirm the intended release candidate on `dev` has green core and specialized CI, including CodeQL.
-2. Convert the relevant `Unreleased` changelog entries into a dated release section and call out any breaking 0.x change explicitly.
-3. Set the release version in both `pyproject.toml` and `src/framevitals/__init__.py` and confirm they match.
-4. Build distributions from the release candidate and run `python -m twine check dist/*`.
-5. Inspect the wheel boundary: the `framevitals` package and `py.typed` must be present, while removed compatibility namespaces and repository-only files must not leak into the wheel.
-6. Install the built wheel into a clean environment and verify at minimum:
-   - `import framevitals`;
-   - installed metadata version equals `framevitals.__version__`;
-   - `framevitals --version` reports the release version.
-7. Merge the reviewed release pull request to `main` only after its required checks are green.
+If the repository is renamed, update the PyPI Trusted Publisher configuration before publishing again.
 
 ## Publish checklist
 
-1. Confirm the release commit is on `main` and the release version/changelog are final.
-2. Create a GitHub release from that exact `main` commit with a matching tag such as `v0.2.0`.
-3. Publishing the GitHub release triggers `.github/workflows/release.yml`.
-4. The release workflow verifies that the GitHub tag, `pyproject.toml`, and `framevitals.__version__` agree before building.
-5. Confirm the Trusted Publishing job succeeds and the expected files appear on PyPI.
-6. Install the published package in a brand-new environment and run:
+1. Promote the reviewed release candidate to `main` through the normal branch process.
+2. Verify the exact `main` commit has the intended release versions and changelog.
+3. Create a GitHub release from that exact commit with a matching tag such as `v0.2.0`.
+4. Publishing the GitHub release triggers `.github/workflows/release.yml`.
+5. The workflow must:
+   - verify tag/Python/Rust version consistency;
+   - build one fallback wheel and one source distribution;
+   - build the configured native ABI3 wheels;
+   - verify each native wheel actually contains `framevitals._native`;
+   - run `twine check` across the complete payload;
+   - publish through Trusted Publishing only after all build jobs pass.
+6. Confirm the expected files appear on PyPI.
+7. In a brand-new environment, run:
 
    ```bash
+   python -m pip install --upgrade framevitals
    python -c "import framevitals; print(framevitals.__version__)"
    framevitals --version
    ```
 
-7. Smoke-test at least one normal analysis path from the published wheel. For a minor release that changes optional capabilities, also smoke-test the affected extra in a clean environment.
+8. On a supported native platform, also verify:
 
-## Version consistency
+   ```bash
+   python -c "from framevitals.backends import backend_status; print(backend_status())"
+   ```
 
-Until version metadata is centralized, keep these two values identical:
+   `native_available` should be `True` and automatic backend selection should choose `rust`.
+9. Smoke-test one normal analysis and one source-aware/Arrow path from the published wheel.
 
-- `pyproject.toml` → `[project].version`
-- `src/framevitals/__init__.py` → `__version__`
+## Release discipline
 
-Do not bump the version merely to merge a development/stabilization branch into `dev`. Bump it only as part of intentional release preparation.
+Do not bump versions merely to merge a stabilization branch into `dev`. Version bumps belong to an intentional release-candidate commit.
 
-A future cleanup can derive the package version from one canonical source.
+Do not publish a release because the feature list is long. Publish when correctness, packaging, provenance, platform compatibility, and performance-sensitive behavior have all passed their release gates.
diff --git a/pyproject.toml b/pyproject.toml
index 4404f5d..f661a15 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "framevitals"
-version = "0.1.0"
+version = "0.2.0"
 description = "Data quality, drift detection, anomaly analysis, and ML-readiness diagnostics for pandas and tabular data."
 readme = "README.md"
 requires-python = ">=3.11"
diff --git a/rust/framevitals-core/Cargo.toml b/rust/framevitals-core/Cargo.toml
index caebf74..3a04db4 100644
--- a/rust/framevitals-core/Cargo.toml
+++ b/rust/framevitals-core/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "framevitals-core"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 description = "Native streaming kernels for FrameVitals"
 license = "MIT"
diff --git a/rust/framevitals-py/Cargo.toml b/rust/framevitals-py/Cargo.toml
index ca137aa..3c5325a 100644
--- a/rust/framevitals-py/Cargo.toml
+++ b/rust/framevitals-py/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "framevitals-py"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 description = "PyO3 bridge for FrameVitals native kernels"
 license = "MIT"
diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py
index ed6b8fc..4039a1a 100644
--- a/src/framevitals/__init__.py
+++ b/src/framevitals/__init__.py
@@ -25,7 +25,7 @@
     from framevitals.checks import DataCheck
     from framevitals.cleaning_plan import CleaningPlan
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 
 
 def __getattr__(name: str):
@@ -336,44 +336,11 @@ def gate(
 
 
 __all__ = [
-    "AnalysisConfig",
-    "AnalysisPlan",
-    "AnalysisResult",
-    "AnalysisSnapshot",
-    "SnapshotHistory",
-    "CleaningPlan",
-    "ColumnResult",
-    "DiagnosticResult",
-    "DataCheck",
-    "CheckResult",
-    "DriftResult",
-    "GateResult",
-    "ValidationResult",
-    "inspect_source",
-    "profile",
-    "roles",
-    "health",
-    "ml_readiness",
-    "quality",
-    "statistics",
-    "anomalies",
-    "relationships",
-    "system_info",
-    "target_analysis",
-    "analyze",
-    "plan",
-    "plan_cleaning",
-    "clean",
-    "compare",
-    "infer_contract",
-    "validate",
-    "check",
-    "run_checks",
-    "discover_checks",
-    "gate",
-    "available_modules",
-    "create_snapshot",
-    "load_snapshot",
-    "compare_snapshots",
-    "__version__",
+    "AnalysisConfig", "AnalysisPlan", "AnalysisResult", "AnalysisSnapshot", "SnapshotHistory",
+    "CleaningPlan", "ColumnResult", "DiagnosticResult", "DataCheck", "CheckResult", "DriftResult",
+    "GateResult", "ValidationResult", "inspect_source", "profile", "roles", "health",
+    "ml_readiness", "quality", "statistics", "anomalies", "relationships", "system_info",
+    "target_analysis", "analyze", "plan", "plan_cleaning", "clean", "compare", "infer_contract",
+    "validate", "check", "run_checks", "discover_checks", "gate", "available_modules",
+    "create_snapshot", "load_snapshot", "compare_snapshots", "__version__",
 ]

From 4dac3263abe3695bcb45acc9088e435e7f9ff32a Mon Sep 17 00:00:00 2001
From: FrameVitals Release Bot <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 07:39:37 +0000
Subject: [PATCH 426/439] fix: harden 0.2.0 release contracts

---
 src/framevitals/ai_agent.py                |  5 +--
 src/framevitals/cleaner.py                 |  1 -
 src/framevitals/explainability.py          |  1 -
 src/framevitals/fast_deep_statistics.py    |  1 -
 src/framevitals/frontend_api.py            |  4 +-
 src/framevitals/operations.py              |  1 -
 src/framevitals/pdf_report_builder.py      | 25 ++++++-----
 src/framevitals/profiler.py                | 13 ++++--
 src/framevitals/target_intelligence.py     |  1 -
 src/framevitals/time_series.py             |  1 -
 src/framevitals/visualizer.py              |  6 +--
 tests/test_extreme_streaming_projection.py |  1 -
 tests/test_public_api.py                   |  4 +-
 tests/test_release_hardening.py            | 48 ++++++++++++++++++++++
 14 files changed, 79 insertions(+), 33 deletions(-)
 create mode 100644 tests/test_release_hardening.py

diff --git a/src/framevitals/ai_agent.py b/src/framevitals/ai_agent.py
index 59383e2..cc9fab9 100644
--- a/src/framevitals/ai_agent.py
+++ b/src/framevitals/ai_agent.py
@@ -1,5 +1,5 @@
 """
-DataLens AI — Agentic AI Layer (WS-9)
+FrameVitals — Agentic AI Layer (WS-9)
 ======================================
 Planner → Executor → Critic → Writer loop running on a local Ollama model
 with OpenRouter and a deterministic fallback.
@@ -27,7 +27,6 @@
 from typing import Any
 
 import pandas as pd
-import pydantic
 from pydantic import BaseModel, Field, ValidationError
 
 from framevitals.rag_index import (
@@ -54,7 +53,7 @@
 )
 _OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
 _OPENROUTER_SITE_URL = os.environ.get("OPENROUTER_SITE_URL", "http://127.0.0.1:5055")
-_OPENROUTER_APP_NAME = os.environ.get("OPENROUTER_APP_NAME", "DataLens AI")
+_OPENROUTER_APP_NAME = os.environ.get("OPENROUTER_APP_NAME", "FrameVitals")
 
 # Per-call Ollama timeout. Cloud models (deepseek 671b, gpt-oss 120b, kimi)
 # can take 30-60s end-to-end. Override with DATALENS_OLLAMA_TIMEOUT (seconds).
diff --git a/src/framevitals/cleaner.py b/src/framevitals/cleaner.py
index a16cead..b64ca0f 100644
--- a/src/framevitals/cleaner.py
+++ b/src/framevitals/cleaner.py
@@ -1,6 +1,5 @@
 from pathlib import Path
 
-import pandas as pd
 
 from framevitals.cleaning_plan import apply_cleaning_plan, infer_cleaning_plan
 from framevitals.health_score import calculate_health_score
diff --git a/src/framevitals/explainability.py b/src/framevitals/explainability.py
index 07d396d..ac3e4ae 100644
--- a/src/framevitals/explainability.py
+++ b/src/framevitals/explainability.py
@@ -19,7 +19,6 @@
 
 import warnings
 from pathlib import Path
-from typing import Any
 
 import numpy as np
 import pandas as pd
diff --git a/src/framevitals/fast_deep_statistics.py b/src/framevitals/fast_deep_statistics.py
index 606eb77..564c28e 100644
--- a/src/framevitals/fast_deep_statistics.py
+++ b/src/framevitals/fast_deep_statistics.py
@@ -20,7 +20,6 @@
 from scipy import stats
 
 from framevitals.deep_statistics_v2 import (
-    _categorical_column_stats,
     _classify_kurtosis,
     _classify_skew,
     _fit_best_distribution,
diff --git a/src/framevitals/frontend_api.py b/src/framevitals/frontend_api.py
index 6c2274c..e5a68ad 100644
--- a/src/framevitals/frontend_api.py
+++ b/src/framevitals/frontend_api.py
@@ -205,8 +205,8 @@ def build_dashboard_payload(
 
     health_score = result.get("health", {}).get("overall_score", 0)
     health_label = result.get("health", {}).get("label", "Unknown")
-    ml_score = result.get("ml_readiness", {}).get("score", 0)
-    ml_label = result.get("ml_readiness", {}).get("label", "Unknown")
+    result.get("ml_readiness", {}).get("score", 0)
+    result.get("ml_readiness", {}).get("label", "Unknown")
 
     payload = {
         "id": result.get("dataset_id"),
diff --git a/src/framevitals/operations.py b/src/framevitals/operations.py
index defb7f9..deb665a 100644
--- a/src/framevitals/operations.py
+++ b/src/framevitals/operations.py
@@ -8,7 +8,6 @@
 from __future__ import annotations
 
 from collections.abc import Mapping, Sequence
-from pathlib import Path
 from typing import Any
 
 import pandas as pd
diff --git a/src/framevitals/pdf_report_builder.py b/src/framevitals/pdf_report_builder.py
index 76396ce..45b6f8f 100644
--- a/src/framevitals/pdf_report_builder.py
+++ b/src/framevitals/pdf_report_builder.py
@@ -1,7 +1,7 @@
 """
 PDF Report Builder (v3 — editorial dark)
 ========================================
-Generates the DataLens AI dataset report PDF.
+Generates the FrameVitals dataset report PDF.
 
 Visual identity matches the dashboard: cream + teal on near-black, heavy
 display weights, mono eyebrows, generous whitespace. Pages flow as:
@@ -238,7 +238,7 @@ def _draw_header(self):
         # Eyebrow brand
         self.lay.text(
             MARGIN_L, PAGE_H - 0.42,
-            "DATALENS  ·  AI DATASET REPORT",
+            "FRAMEVITALS  ·  DATASET REPORT",
             fontsize=8, color=INK_3, fontweight="bold",
             family="monospace", va="center", ha="left",
         )
@@ -267,7 +267,7 @@ def _draw_footer(self):
         ))
         self.lay.text(
             MARGIN_L, MARGIN_B - 0.32,
-            "DATALENS · AI",
+            "FRAMEVITALS",
             fontsize=8, color=INK_3, fontweight="bold",
             family="monospace", va="center", ha="left",
         )
@@ -624,7 +624,6 @@ def add_chart_image(self, chart: dict, *, x: float, y_top: float,
 
 def _cover_page(c: Composer, result: dict):
     c.new_page(cover=True, chrome=False)
-    f = c.fig
     lay = c.lay
 
     # Hero band — gradient from teal-soft → near-black using a stack of rects
@@ -639,7 +638,7 @@ def _cover_page(c: Composer, result: dict):
     # Brand eyebrow
     lay.text(
         MARGIN_L, PAGE_H - 1.0,
-        "DATALENS · AI",
+        "FRAMEVITALS",
         fontsize=10, color=ACCENT, fontweight="bold",
         family="monospace", ha="left", va="top",
     )
@@ -733,7 +732,7 @@ def _color_for_score(v):
     c.y = MARGIN_B + 1.6
     lay.text(
         MARGIN_L, c.y,
-        "Generated by DataLens AI's analytical pipeline.",
+        "Generated by FrameVitals' analytical pipeline.",
         fontsize=10, color=INK_3, fontweight="700",
         ha="left", va="top",
     )
@@ -748,7 +747,7 @@ def _color_for_score(v):
     c.hline(MARGIN_L, MARGIN_L + CONTENT_W, MARGIN_B + 0.4, color=LINE, lw=0.6)
     lay.text(
         MARGIN_L, MARGIN_B + 0.18,
-        "DATALENS · AI",
+        "FRAMEVITALS",
         fontsize=8, color=INK_3, fontweight="bold",
         family="monospace", ha="left", va="top",
     )
@@ -830,7 +829,7 @@ def _render_executive_summary(c: Composer, result: dict):
     lead = (
         f"This dataset contains {rows_v} rows across {cols_v} columns. "
         f"It scores {health_v}/100 on quality ({health_label}) and {ml_v}/100 on ML readiness ({ml_label}). "
-        f"DataLens AI ran in {str(mode).upper()} mode and selected "
+        f"FrameVitals ran in {str(mode).upper()} mode and selected "
         f"{sel.get('selected_count', 0)} analyses, recommended {sel.get('recommended_count', 0)} more, "
         f"and skipped {sel.get('skipped_count', 0)}."
     )
@@ -929,7 +928,7 @@ def _render_cleaning(c: Composer, result: dict):
         return
     c.new_page()
     c.section("Cleaning Summary", kind="cleaning",
-              subtitle="What changed when DataLens auto-cleaned the dataset.")
+              subtitle="Summary of applied cleaning actions and health impact.")
 
     before = (cl.get("before_health") or {}).get("overall_score")
     after = (cl.get("after_health") or {}).get("overall_score")
@@ -1094,7 +1093,7 @@ def _render_ai_report(c: Composer, result: dict):
 
     c.new_page()
     c.section("AI Analyst Report", kind="ai",
-              subtitle="Narrative interpretation of the dataset by DataLens's analyst model.")
+              subtitle="Narrative interpretation of the dataset by FrameVitals' analyst model.")
     c.callout(f"source: {ai.get('source', 'unknown')}", color=ACCENT_2)
     c.text_block(text[:8000], size=10, color=INK_2, weight="700", line_height=0.20)
 
@@ -1162,7 +1161,7 @@ def _build_toc_sections(result: dict) -> list[tuple[str, str, str]]:
                           "Auto-detected issues, sorted by severity."))
     if result.get("cleaning"):
         sections.append((f"{len(sections)+1:02d}", "Cleaning Summary",
-                          "Before / after health and the actions DataLens took."))
+                          "Before / after health and applied cleaning actions."))
     if result.get("deep_statistics_v2"):
         sections.append((f"{len(sections)+1:02d}", "Deep Statistics",
                           "Pairwise relationships and group differences."))
@@ -1178,7 +1177,7 @@ def _build_toc_sections(result: dict) -> list[tuple[str, str, str]]:
     ai = result.get("ai_report") or {}
     if ai.get("text"):
         sections.append((f"{len(sections)+1:02d}", "AI Analyst Report",
-                          "Narrative interpretation by the DataLens analyst model."))
+                          "Narrative interpretation by the FrameVitals analyst model."))
     if result.get("charts"):
         sections.append((f"{len(sections)+1:02d}", "Visual Evidence",
                           f"{len(result.get('charts') or [])} planner-driven charts, two per page."))
@@ -1188,7 +1187,7 @@ def _build_toc_sections(result: dict) -> list[tuple[str, str, str]]:
 def generate_pdf_report(
     result: dict,
     output_dir: Path | str = "reports",
-    report_title: str = "DataLens AI Dataset Report",
+    report_title: str = "FrameVitals Dataset Report",
 ) -> Path:
     output_dir = Path(output_dir)
     output_dir.mkdir(parents=True, exist_ok=True)
diff --git a/src/framevitals/profiler.py b/src/framevitals/profiler.py
index a5e09e1..31c9160 100644
--- a/src/framevitals/profiler.py
+++ b/src/framevitals/profiler.py
@@ -6,6 +6,7 @@
 import pandas as pd
 
 from framevitals.backends import numeric_profile, resolve_numeric_backend
+from framevitals.execution import _deterministic_stratified_positions
 
 MAX_CORRELATION_COLUMNS = 100
 MAX_EXACT_DUPLICATE_CELLS = 50_000_000
@@ -77,8 +78,8 @@ def _duplicate_profile(df: pd.DataFrame) -> tuple[int, dict]:
         }
 
     sample_rows = min(rows, DUPLICATE_SAMPLE_ROWS)
-    positions = np.linspace(0, rows - 1, num=sample_rows, dtype=np.int64)
-    sample = df.iloc[np.unique(positions)]
+    positions = _deterministic_stratified_positions(rows, sample_rows)
+    sample = df.iloc[positions]
     sample_duplicates = int(sample.duplicated().sum())
     rate = sample_duplicates / max(len(sample), 1)
     estimate = int(round(rate * rows))
@@ -88,6 +89,7 @@ def _duplicate_profile(df: pd.DataFrame) -> tuple[int, dict]:
         "sample_rows": int(len(sample)),
         "source_rows": int(rows),
         "estimated_duplicate_rate": round(float(rate), 6),
+        "strategy": "stratified_jitter_global_rows",
     }
 
 
@@ -115,8 +117,9 @@ def _bounded_correlations(
             key=lambda column: (-non_missing[column], str(column)),
         )[:MAX_CORRELATION_COLUMNS]
 
+    finite_numeric = df[selected].replace([np.inf, -np.inf], np.nan)
     correlations = (
-        df[selected]
+        finite_numeric
         .corr(numeric_only=True)
         .round(3)
         .replace({np.nan: None})
@@ -148,8 +151,9 @@ def _pandas_numeric_summary(
             "approximate_quantiles": False,
         }, {}
 
+    finite_numeric = df[numeric_cols].replace([np.inf, -np.inf], np.nan)
     summary = (
-        df[numeric_cols]
+        finite_numeric
         .describe()
         .T
         .replace({np.nan: None})
@@ -160,6 +164,7 @@ def _pandas_numeric_summary(
         "backend": "pandas",
         "method": "pandas_describe",
         "approximate_quantiles": False,
+        "finite_only_moments": True,
         "columns_profiled": len(numeric_cols),
     }, {}
 
diff --git a/src/framevitals/target_intelligence.py b/src/framevitals/target_intelligence.py
index 5f510de..9cca085 100644
--- a/src/framevitals/target_intelligence.py
+++ b/src/framevitals/target_intelligence.py
@@ -10,7 +10,6 @@
 import math
 from typing import Any
 
-import numpy as np
 import pandas as pd
 from scipy import stats
 
diff --git a/src/framevitals/time_series.py b/src/framevitals/time_series.py
index 5b88311..8de00a8 100644
--- a/src/framevitals/time_series.py
+++ b/src/framevitals/time_series.py
@@ -243,7 +243,6 @@ def _guess_period_via_fft(values: np.ndarray, freq_label: str) -> int | None:
 # ---------------------------------------------------------------------------
 
 def _stationarity(series: pd.Series) -> dict:
-    out: dict[str, Any] = {"available": False}
     s = series.dropna()
     if len(s) < 30:
         return {"available": False, "reason": "n<30"}
diff --git a/src/framevitals/visualizer.py b/src/framevitals/visualizer.py
index 9d23597..f6c1fbb 100644
--- a/src/framevitals/visualizer.py
+++ b/src/framevitals/visualizer.py
@@ -12,7 +12,7 @@
 matplotlib.use("Agg")
 
 import matplotlib.pyplot as plt
-from matplotlib.patches import Wedge, Patch
+from matplotlib.patches import Patch
 import seaborn as sns
 import numpy as np
 import pandas as pd
@@ -484,7 +484,7 @@ def _chart_pareto_categorical(did, df, col):
     cum = counts.cumsum() / counts.sum() * 100
 
     fig, ax1 = plt.subplots(figsize=(10, 5.2))
-    bars = ax1.bar(range(len(counts)), counts.values, color=ACCENT,
+    ax1.bar(range(len(counts)), counts.values, color=ACCENT,
                    edgecolor=BG_PAGE, linewidth=1.0, width=0.65, zorder=3)
     ax1.set_xticks(range(len(counts)))
     ax1.set_xticklabels([str(x)[:18] for x in counts.index],
@@ -1020,7 +1020,7 @@ def generate_charts(
                 chart = _chart_time_series_trend(dataset_id, df, time_series)
             elif t == "bivariate_highlights":
                 chart = _chart_bivariate_highlights(dataset_id, deep_statistics_v2)
-        except Exception as exc:  # noqa: BLE001 — never let a single chart kill the run
+        except Exception:  # noqa: BLE001 — never let a single chart kill the run
             chart = None
             # We deliberately don't log here at WARN level because chart fallout
             # is expected (e.g. all-NaN columns, single-class targets, etc.).
diff --git a/tests/test_extreme_streaming_projection.py b/tests/test_extreme_streaming_projection.py
index e77add0..f480390 100644
--- a/tests/test_extreme_streaming_projection.py
+++ b/tests/test_extreme_streaming_projection.py
@@ -1,7 +1,6 @@
 from types import SimpleNamespace
 
 import numpy as np
-import pandas as pd
 import pytest
 
 pa = pytest.importorskip("pyarrow")
diff --git a/tests/test_public_api.py b/tests/test_public_api.py
index 4e1b745..b3f8658 100644
--- a/tests/test_public_api.py
+++ b/tests/test_public_api.py
@@ -1,3 +1,5 @@
+from importlib.metadata import version as distribution_version
+
 import pandas as pd
 import pytest
 
@@ -5,7 +7,7 @@
 
 
 def test_package_version():
-    assert framevitals.__version__ == "0.1.0"
+    assert framevitals.__version__ == distribution_version("framevitals")
 
 
 def test_analyze_is_public():
diff --git a/tests/test_release_hardening.py b/tests/test_release_hardening.py
new file mode 100644
index 0000000..f9166f5
--- /dev/null
+++ b/tests/test_release_hardening.py
@@ -0,0 +1,48 @@
+import inspect
+import warnings
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from framevitals import profiler
+from framevitals.profiler import build_profile
+
+
+def test_materialized_duplicate_estimate_uses_stratified_jitter(monkeypatch):
+    frame = pd.DataFrame({"value": [1, 1, 2, 2, 3, 3, 4, 4]})
+    seen = {}
+    def positions(rows, target_rows):
+        seen["args"] = (rows, target_rows)
+        return np.array([0, 2, 4, 6], dtype=np.int64)
+    monkeypatch.setattr(profiler, "MAX_EXACT_DUPLICATE_CELLS", 1)
+    monkeypatch.setattr(profiler, "DUPLICATE_SAMPLE_ROWS", 4)
+    monkeypatch.setattr(profiler, "_deterministic_stratified_positions", positions)
+    _, metadata = profiler._duplicate_profile(frame)
+    assert seen["args"] == (len(frame), 4)
+    assert metadata["sampled"] is True
+    assert metadata["strategy"] == "stratified_jitter_global_rows"
+
+
+def test_pandas_profile_excludes_infinities_from_finite_moments_without_hiding_missingness(monkeypatch):
+    monkeypatch.setenv("FRAMEVITALS_BACKEND", "numpy")
+    frame = pd.DataFrame({"x": [1.0, 2.0, np.inf, -np.inf, np.nan], "y": [2.0, 4.0, 8.0, 16.0, 32.0]})
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
+        profile = build_profile(frame)
+    x = profile["numeric_summary"]["x"]
+    assert x["count"] == 2.0
+    assert x["mean"] == pytest.approx(1.5)
+    assert x["min"] == pytest.approx(1.0)
+    assert x["max"] == pytest.approx(2.0)
+    assert profile["missing_counts"]["x"] == 1
+    assert profile["numeric_summary_metadata"]["finite_only_moments"] is True
+    assert profile["correlations"]["x"]["y"] == pytest.approx(1.0)
+
+
+def test_pdf_report_builder_contains_only_framevitals_branding():
+    import framevitals.pdf_report_builder as report_builder
+    source = inspect.getsource(report_builder)
+    assert "DataLens" not in source
+    assert "DATALENS" not in source
+    assert "FrameVitals Dataset Report" in source

From 4032f4d57b16c2a49a99a3e20360e0305a562f76 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 13:10:15 +0530
Subject: [PATCH 427/439] ci: make Python dead-code lint blocking

---
 .github/workflows/test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c20b566..3892b7d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -41,8 +41,7 @@ jobs:
       - name: Check undefined-name and scope errors
         run: ruff check src/framevitals tests --select F821,F822,F823
 
-      - name: Report broader dead-code lint debt
-        continue-on-error: true
+      - name: Check unused imports and assignments
         run: ruff check src/framevitals tests --select F401,F841
 
   core:

From ed1bb130c996f83920ad350bfec0feb9664a379b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 13:12:23 +0530
Subject: [PATCH 428/439] ci: include release hardening in backend parity gate

---
 .github/workflows/mixed-ground-truth.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/mixed-ground-truth.yml b/.github/workflows/mixed-ground-truth.yml
index 56833cf..6c9f5b9 100644
--- a/.github/workflows/mixed-ground-truth.yml
+++ b/.github/workflows/mixed-ground-truth.yml
@@ -6,6 +6,7 @@ on:
     paths:
       - ".github/workflows/mixed-ground-truth.yml"
       - "tests/test_mixed_ground_truth_pipeline.py"
+      - "tests/test_release_hardening.py"
       - "tests/test_parquet_streaming.py"
       - "tests/test_target_intelligence.py"
       - "tests/test_target_intelligence_v2.py"
@@ -78,6 +79,7 @@ jobs:
         run: |
           pytest -q \
             tests/test_mixed_ground_truth_pipeline.py \
+            tests/test_release_hardening.py \
             tests/test_parquet_streaming.py \
             tests/test_target_intelligence.py \
             tests/test_target_intelligence_v2.py \
@@ -93,6 +95,7 @@ jobs:
         run: |
           pytest -q \
             tests/test_mixed_ground_truth_pipeline.py \
+            tests/test_release_hardening.py \
             tests/test_parquet_streaming.py \
             tests/test_target_intelligence.py \
             tests/test_target_intelligence_v2.py \
@@ -111,7 +114,7 @@ jobs:
 
           print("## Mixed ground-truth validation")
           print()
-          print("Coverage: mixed Parquet streaming, exact missingness/moments, quantile tolerance, categorical/date detection, target intelligence, semantic types, relationship discovery, anomaly recall, and backend parity through the same test contracts.")
+          print("Coverage: mixed Parquet streaming, exact missingness/moments, finite-value fallback semantics, sampling provenance, quantile tolerance, categorical/date detection, target intelligence, semantic types, relationship discovery, anomaly recall, and backend parity through the same test contracts.")
           print()
           print("| Backend | Tests | Failures | Errors | Skipped |")
           print("|---|---:|---:|---:|---:|")

From a17fa85c6c4d95c9de1fd229ea3993309607776b Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 13:12:42 +0530
Subject: [PATCH 429/439] ci: rerun performance guardrail for release hardening

---
 .github/workflows/performance-guardrail.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/performance-guardrail.yml b/.github/workflows/performance-guardrail.yml
index 2e00bf4..c06f4be 100644
--- a/.github/workflows/performance-guardrail.yml
+++ b/.github/workflows/performance-guardrail.yml
@@ -9,6 +9,7 @@ on:
       - "src/framevitals/sources.py"
       - "src/framevitals/focused.py"
       - "src/framevitals/native_core.py"
+      - "tests/test_release_hardening.py"
       - "rust/**"
       - "benchmarks/performance_guardrail.py"
       - "benchmarks/performance_budgets.json"
@@ -21,6 +22,7 @@ on:
       - "src/framevitals/sources.py"
       - "src/framevitals/focused.py"
       - "src/framevitals/native_core.py"
+      - "tests/test_release_hardening.py"
       - "rust/**"
       - "benchmarks/performance_guardrail.py"
       - "benchmarks/performance_budgets.json"

From 4616b9e53dbd9467a9b44c42092bba70970b1654 Mon Sep 17 00:00:00 2001
From: FrameVitals Release Bot <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 07:48:23 +0000
Subject: [PATCH 430/439] build: pin Rust dependencies for 0.2.0

---
 rust/Cargo.lock | 957 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 957 insertions(+)
 create mode 100644 rust/Cargo.lock

diff --git a/rust/Cargo.lock b/rust/Cargo.lock
new file mode 100644
index 0000000..acd3556
--- /dev/null
+++ b/rust/Cargo.lock
@@ -0,0 +1,957 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "ahash"
+version = "0.8.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
+dependencies = [
+ "cfg-if",
+ "const-random",
+ "getrandom 0.3.4",
+ "once_cell",
+ "version_check",
+ "zerocopy",
+]
+
+[[package]]
+name = "android_system_properties"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "arrow-array"
+version = "59.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97"
+dependencies = [
+ "ahash",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "chrono",
+ "chrono-tz",
+ "half",
+ "hashbrown",
+ "libc",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "arrow-buffer"
+version = "59.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90"
+dependencies = [
+ "bytes",
+ "half",
+ "num-bigint",
+ "num-traits",
+]
+
+[[package]]
+name = "arrow-cast"
+version = "59.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-ord",
+ "arrow-schema",
+ "arrow-select",
+ "atoi",
+ "base64",
+ "chrono",
+ "comfy-table",
+ "half",
+ "lexical-core",
+ "num-traits",
+ "ryu",
+]
+
+[[package]]
+name = "arrow-data"
+version = "59.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d"
+dependencies = [
+ "arrow-buffer",
+ "arrow-schema",
+ "half",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "arrow-ord"
+version = "59.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "arrow-select",
+]
+
+[[package]]
+name = "arrow-schema"
+version = "59.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "arrow-select"
+version = "59.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb"
+dependencies = [
+ "ahash",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "num-traits",
+]
+
+[[package]]
+name = "atoi"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "base64"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
+
+[[package]]
+name = "bitflags"
+version = "2.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
+
+[[package]]
+name = "bumpalo"
+version = "3.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
+
+[[package]]
+name = "bytes"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
+
+[[package]]
+name = "cc"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
+dependencies = [
+ "find-msvc-tools",
+ "shlex",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "chrono"
+version = "0.4.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
+dependencies = [
+ "iana-time-zone",
+ "js-sys",
+ "num-traits",
+ "wasm-bindgen",
+ "windows-link",
+]
+
+[[package]]
+name = "chrono-tz"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3"
+dependencies = [
+ "chrono",
+ "phf",
+]
+
+[[package]]
+name = "comfy-table"
+version = "7.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47"
+dependencies = [
+ "unicode-segmentation",
+ "unicode-width",
+]
+
+[[package]]
+name = "const-random"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
+dependencies = [
+ "const-random-macro",
+]
+
+[[package]]
+name = "const-random-macro"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
+dependencies = [
+ "getrandom 0.2.17",
+ "once_cell",
+ "tiny-keccak",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+
+[[package]]
+name = "crunchy"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+
+[[package]]
+name = "equivalent"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
+
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890"
+
+[[package]]
+name = "framevitals-core"
+version = "0.2.0"
+dependencies = [
+ "arrow-array",
+ "arrow-schema",
+ "rustc-hash",
+]
+
+[[package]]
+name = "framevitals-py"
+version = "0.2.0"
+dependencies = [
+ "framevitals-core",
+ "pyo3",
+ "pyo3-arrow",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
+
+[[package]]
+name = "futures-task"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
+
+[[package]]
+name = "futures-util"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "pin-project-lite",
+ "slab",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasip2",
+]
+
+[[package]]
+name = "half"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+ "num-traits",
+ "zerocopy",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "iana-time-zone"
+version = "0.1.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "log",
+ "wasm-bindgen",
+ "windows-core",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+]
+
+[[package]]
+name = "js-sys"
+version = "0.3.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a"
+dependencies = [
+ "cfg-if",
+ "futures-util",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "lexical-core"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594"
+dependencies = [
+ "lexical-parse-float",
+ "lexical-parse-integer",
+ "lexical-util",
+ "lexical-write-float",
+ "lexical-write-integer",
+]
+
+[[package]]
+name = "lexical-parse-float"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56"
+dependencies = [
+ "lexical-parse-integer",
+ "lexical-util",
+]
+
+[[package]]
+name = "lexical-parse-integer"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34"
+dependencies = [
+ "lexical-util",
+]
+
+[[package]]
+name = "lexical-util"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17"
+
+[[package]]
+name = "lexical-write-float"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361"
+dependencies = [
+ "lexical-util",
+ "lexical-write-integer",
+]
+
+[[package]]
+name = "lexical-write-integer"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df"
+dependencies = [
+ "lexical-util",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.189"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
+
+[[package]]
+name = "libm"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
+
+[[package]]
+name = "log"
+version = "0.4.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+
+[[package]]
+name = "matrixmultiply"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7"
+dependencies = [
+ "autocfg",
+ "rawpointer",
+]
+
+[[package]]
+name = "ndarray"
+version = "0.17.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
+dependencies = [
+ "matrixmultiply",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "portable-atomic",
+ "portable-atomic-util",
+ "rawpointer",
+]
+
+[[package]]
+name = "num-bigint"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-complex"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+ "libm",
+]
+
+[[package]]
+name = "numpy"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a5b15d63a5ff39e378daed0e1340d3a5964703ea9712eb09a0dc66fade996f4"
+dependencies = [
+ "half",
+ "libc",
+ "ndarray",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "pyo3",
+ "pyo3-build-config",
+ "rustc-hash",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "phf"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7"
+dependencies = [
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618"
+dependencies = [
+ "portable-atomic",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b"
+dependencies = [
+ "chrono",
+ "chrono-tz",
+ "indexmap",
+ "libc",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+]
+
+[[package]]
+name = "pyo3-arrow"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d5ddf226a2dbf7607570d0657c2bf6fbe299208368b2914f3dd7e7ba0b57688"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-data",
+ "arrow-schema",
+ "arrow-select",
+ "chrono",
+ "chrono-tz",
+ "half",
+ "indexmap",
+ "numpy",
+ "pyo3",
+ "thiserror",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
+[[package]]
+name = "rawpointer"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
+
+[[package]]
+name = "rustc-hash"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
+
+[[package]]
+name = "rustversion"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
+
+[[package]]
+name = "ryu"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
+
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
+[[package]]
+name = "siphasher"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
+
+[[package]]
+name = "slab"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tiny-keccak"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
+dependencies = [
+ "crunchy",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
+
+[[package]]
+name = "unicode-width"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
+
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "wasi"
+version = "0.11.1+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
+
+[[package]]
+name = "wasip2"
+version = "1.0.4+wasi-0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.127"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "rustversion",
+ "wasm-bindgen-macro",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.127"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.127"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284"
+dependencies = [
+ "bumpalo",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.127"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.62.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
+dependencies = [
+ "windows-implement",
+ "windows-interface",
+ "windows-link",
+ "windows-result",
+ "windows-strings",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.60.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.59.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "windows-result"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "wit-bindgen"
+version = "0.57.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
+
+[[package]]
+name = "zerocopy"
+version = "0.8.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]

From e9d880913e80d9333c8c5488a21c62a0328e0a9f Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 13:18:59 +0530
Subject: [PATCH 431/439] ci: enforce locked Rust dependencies in package
 builds

---
 .github/workflows/package.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index b268fbf..140857d 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -44,6 +44,7 @@ jobs:
       - name: Build native wheel
         run: |
           maturin build \
+            --locked \
             --release \
             --manifest-path rust/framevitals-py/Cargo.toml \
             --interpreter python \

From 05b11e594995a3833ec08f5b4a8a145197bf4cab Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 13:19:17 +0530
Subject: [PATCH 432/439] ci: enforce locked Rust dependencies in PyPI release

---
 .github/workflows/release.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index dd19c76..da73af8 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -48,6 +48,7 @@ jobs:
           with Path("rust/framevitals-py/Cargo.toml").open("rb") as handle:
               bridge_version = tomllib.load(handle)["package"]["version"]
           package_version = framevitals.__version__
+          assert Path("rust/Cargo.lock").is_file(), "Rust workspace lockfile is missing"
 
           versions = {
               "release tag": release_version,
@@ -116,6 +117,7 @@ jobs:
           target: ${{ matrix.target }}
           manylinux: ${{ matrix.manylinux }}
           args: >-
+            --locked
             --release
             --manifest-path rust/framevitals-py/Cargo.toml
             --interpreter python

From 2c3b327db924e86f8e161c29b823bade6dc9c533 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 14:12:00 +0530
Subject: [PATCH 433/439] bench: record 0.2.0 release performance delta

---
 CHANGELOG.md                                  |  15 +
 .../release_0.2.0_vs_0.1.0_10k_x64.json       | 322 ++++++++++++++++++
 tests/test_benchmark_harness.py               |  98 ++++++
 3 files changed, 435 insertions(+)
 create mode 100644 benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d22c936..037f0b4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -62,6 +62,21 @@ FrameVitals benchmarks are not advertised as universal speedups. Results depend
 
 Validated physical workloads include 1B, 2.5B, and 5B logical cells. On the 5B 500k × 10k workload, the native full-stream profiling kernel remains competitive while retaining full-stream native quantile sketches where the fallback may choose bounded row-sample quantiles for cost. All published stress workflows enforce no full materialization and correctness tolerances.
 
+### Same-dataset release comparison vs 0.1.0
+
+For an end-to-end release comparison, `v0.1.0` (`3da1432`) and the 0.2.0 release candidate (`05b11e5`) were run through the same public `framevitals.analyze(path, mode=..., artifacts=False)` API on one deterministic physical 10,000 × 64 CSV (640,000 cells). Both environments used Python 3.11 with the same pinned NumPy/pandas/SciPy/statsmodels/scikit-learn stack. FrameVitals 0.2.0 used its native Rust backend. Each version/mode received one warm-up followed by three measured runs in interleaved ABBAAB order; all 10,000 rows and all 64 columns were validated in every run.
+
+| Mode | 0.1.0 median wall | 0.2.0 median wall | Speedup | 0.1.0 median peak RSS | 0.2.0 median peak RSS | Memory change |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| Quick | 1.644 s | 0.691 s | **2.38×** | 234.7 MB | 239.5 MB | 2.1% higher |
+| Standard | 141.084 s | 0.839 s | **168.06×** | 2736.7 MB | 264.9 MB | **90.3% lower** |
+
+The measured Quick ranges were 1.620–1.648 s for 0.1.0 and 0.686–0.692 s for 0.2.0. Standard ranges were 140.709–141.130 s for 0.1.0 and 0.823–0.841 s for 0.2.0. The large Standard improvement is a release-level architectural result: 0.2.0 replaces 0.1.0's fully materialized, unbounded heavy-statistics path with bounded/adaptive execution and native streaming. It should not be interpreted as a same-algorithm microbenchmark.
+
+The machine-readable evidence is committed as `benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json` from GitHub Actions run `32010158292`.
+
+A separate 100,000 × 64 (6.4M-cell) stress run completed all Quick repetitions at a 5.097 s vs 1.447 s median (**3.52× faster**) and 406.1 MB vs 282.7 MB median peak RSS (**30.4% lower**) for 0.1.0 vs 0.2.0. The repeated Standard comparison intentionally has no formal speedup claim: the 30-minute workflow limit expired because completed 0.1.0 Standard passes took about 640–649 s and roughly 12.4 GB peak RSS each, while observed 0.2.0 Standard passes completed in about 1.58–2.06 s at roughly 303–308 MB. That incomplete stress evidence is GitHub Actions run `32008727705`.
+
 ### Packaging and compatibility
 
 - Added Python 3.11, 3.12, and 3.13 core coverage plus Windows/macOS smoke testing.
diff --git a/benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json b/benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json
new file mode 100644
index 0000000..3640920
--- /dev/null
+++ b/benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json
@@ -0,0 +1,322 @@
+{
+  "benchmark_schema_version": 1,
+  "comparison": "FrameVitals 0.2.0 vs 0.1.0",
+  "old_ref": "v0.1.0@3da1432168fbfcb3dbe99fcfb6f6200f5e63214b",
+  "new_ref": "develop/august@05b11e594995a3833ec08f5b4a8a145197bf4cab",
+  "dataset": {
+    "rows": 10000,
+    "columns": 64,
+    "cells": 640000,
+    "format": "csv",
+    "bytes": 2811188,
+    "c000": {
+      "count": 10000,
+      "mean": -1.4965,
+      "min": -1000,
+      "max": 998
+    }
+  },
+  "methodology": {
+    "runner": "GitHub Actions ubuntu-latest",
+    "python": "3.11",
+    "warmups_per_version_mode": 1,
+    "measured_repetitions_per_version_mode": 3,
+    "measurement_order": "ABBAAB",
+    "public_api": "framevitals.analyze(path, mode=..., artifacts=False)",
+    "scope": "all 10,000 rows and all 64 columns in both releases",
+    "dependency_stack": {
+      "numpy": "1.26.0",
+      "pandas": "2.2.0",
+      "scipy": "1.13.0",
+      "statsmodels": "0.14.0",
+      "scikit-learn": "1.5.0",
+      "pyarrow": "25.0.0"
+    }
+  },
+  "modes": {
+    "quick": {
+      "0.1.0": {
+        "wall_seconds": [
+          1.6478449049999995,
+          1.619613931999993,
+          1.6437554540000008
+        ],
+        "median_wall_seconds": 1.6437554540000008,
+        "min_wall_seconds": 1.619613931999993,
+        "max_wall_seconds": 1.6478449049999995,
+        "peak_rss_mb": [
+          234.6953125,
+          234.5390625,
+          234.6796875
+        ],
+        "median_peak_rss_mb": 234.6796875,
+        "backend": {
+          "selected": "legacy-python",
+          "native_available": false
+        },
+        "profiled_columns": 64,
+        "full_materialization": null
+      },
+      "0.2.0": {
+        "wall_seconds": [
+          0.6861838410000018,
+          0.6908588409999936,
+          0.691870496000007
+        ],
+        "median_wall_seconds": 0.6908588409999936,
+        "min_wall_seconds": 0.6861838410000018,
+        "max_wall_seconds": 0.691870496000007,
+        "peak_rss_mb": [
+          239.72265625,
+          238.875,
+          239.515625
+        ],
+        "median_peak_rss_mb": 239.515625,
+        "backend": {
+          "selected": "rust",
+          "native_available": true,
+          "environment_override": "rust",
+          "eligible": [
+            "numpy",
+            "rust"
+          ]
+        },
+        "profiled_columns": 64,
+        "full_materialization": null
+      },
+      "speedup_x": 2.3792927823297783,
+      "wall_time_reduction_percent": 57.970704260246166,
+      "peak_rss_reduction_percent": -2.0606544825060746
+    },
+    "standard": {
+      "0.1.0": {
+        "wall_seconds": [
+          141.083503924,
+          141.13010660999998,
+          140.708747796
+        ],
+        "median_wall_seconds": 141.083503924,
+        "min_wall_seconds": 140.708747796,
+        "max_wall_seconds": 141.13010660999998,
+        "peak_rss_mb": [
+          2737.171875,
+          2736.421875,
+          2736.7265625
+        ],
+        "median_peak_rss_mb": 2736.7265625,
+        "backend": {
+          "selected": "legacy-python",
+          "native_available": false
+        },
+        "profiled_columns": 64,
+        "full_materialization": null
+      },
+      "0.2.0": {
+        "wall_seconds": [
+          0.8411333409999884,
+          0.822897117000025,
+          0.8394920780000348
+        ],
+        "median_wall_seconds": 0.8394920780000348,
+        "min_wall_seconds": 0.822897117000025,
+        "max_wall_seconds": 0.8411333409999884,
+        "peak_rss_mb": [
+          265.33984375,
+          264.4296875,
+          264.8984375
+        ],
+        "median_peak_rss_mb": 264.8984375,
+        "backend": {
+          "selected": "rust",
+          "native_available": true,
+          "environment_override": "rust",
+          "eligible": [
+            "numpy",
+            "rust"
+          ]
+        },
+        "profiled_columns": 64,
+        "full_materialization": null
+      },
+      "speedup_x": 168.05817186519556,
+      "wall_time_reduction_percent": 99.40496794121852,
+      "peak_rss_reduction_percent": 90.32060998969457
+    }
+  },
+  "measurements": [
+    {
+      "version": "0.1.0",
+      "mode": "quick",
+      "elapsed_seconds": 1.6478449049999995,
+      "peak_rss_mb": 234.6953125,
+      "profiled_columns": 64,
+      "backend_status": {
+        "selected": "legacy-python",
+        "native_available": false
+      },
+      "full_materialization": null
+    },
+    {
+      "version": "0.1.0",
+      "mode": "quick",
+      "elapsed_seconds": 1.619613931999993,
+      "peak_rss_mb": 234.5390625,
+      "profiled_columns": 64,
+      "backend_status": {
+        "selected": "legacy-python",
+        "native_available": false
+      },
+      "full_materialization": null
+    },
+    {
+      "version": "0.1.0",
+      "mode": "quick",
+      "elapsed_seconds": 1.6437554540000008,
+      "peak_rss_mb": 234.6796875,
+      "profiled_columns": 64,
+      "backend_status": {
+        "selected": "legacy-python",
+        "native_available": false
+      },
+      "full_materialization": null
+    },
+    {
+      "version": "0.2.0",
+      "mode": "quick",
+      "elapsed_seconds": 0.6861838410000018,
+      "peak_rss_mb": 239.72265625,
+      "profiled_columns": 64,
+      "backend_status": {
+        "selected": "rust",
+        "native_available": true,
+        "environment_override": "rust",
+        "eligible": [
+          "numpy",
+          "rust"
+        ]
+      },
+      "full_materialization": null
+    },
+    {
+      "version": "0.2.0",
+      "mode": "quick",
+      "elapsed_seconds": 0.6908588409999936,
+      "peak_rss_mb": 238.875,
+      "profiled_columns": 64,
+      "backend_status": {
+        "selected": "rust",
+        "native_available": true,
+        "environment_override": "rust",
+        "eligible": [
+          "numpy",
+          "rust"
+        ]
+      },
+      "full_materialization": null
+    },
+    {
+      "version": "0.2.0",
+      "mode": "quick",
+      "elapsed_seconds": 0.691870496000007,
+      "peak_rss_mb": 239.515625,
+      "profiled_columns": 64,
+      "backend_status": {
+        "selected": "rust",
+        "native_available": true,
+        "environment_override": "rust",
+        "eligible": [
+          "numpy",
+          "rust"
+        ]
+      },
+      "full_materialization": null
+    },
+    {
+      "version": "0.1.0",
+      "mode": "standard",
+      "elapsed_seconds": 141.083503924,
+      "peak_rss_mb": 2737.171875,
+      "profiled_columns": 64,
+      "backend_status": {
+        "selected": "legacy-python",
+        "native_available": false
+      },
+      "full_materialization": null
+    },
+    {
+      "version": "0.1.0",
+      "mode": "standard",
+      "elapsed_seconds": 141.13010660999998,
+      "peak_rss_mb": 2736.421875,
+      "profiled_columns": 64,
+      "backend_status": {
+        "selected": "legacy-python",
+        "native_available": false
+      },
+      "full_materialization": null
+    },
+    {
+      "version": "0.1.0",
+      "mode": "standard",
+      "elapsed_seconds": 140.708747796,
+      "peak_rss_mb": 2736.7265625,
+      "profiled_columns": 64,
+      "backend_status": {
+        "selected": "legacy-python",
+        "native_available": false
+      },
+      "full_materialization": null
+    },
+    {
+      "version": "0.2.0",
+      "mode": "standard",
+      "elapsed_seconds": 0.8411333409999884,
+      "peak_rss_mb": 265.33984375,
+      "profiled_columns": 64,
+      "backend_status": {
+        "selected": "rust",
+        "native_available": true,
+        "environment_override": "rust",
+        "eligible": [
+          "numpy",
+          "rust"
+        ]
+      },
+      "full_materialization": null
+    },
+    {
+      "version": "0.2.0",
+      "mode": "standard",
+      "elapsed_seconds": 0.822897117000025,
+      "peak_rss_mb": 264.4296875,
+      "profiled_columns": 64,
+      "backend_status": {
+        "selected": "rust",
+        "native_available": true,
+        "environment_override": "rust",
+        "eligible": [
+          "numpy",
+          "rust"
+        ]
+      },
+      "full_materialization": null
+    },
+    {
+      "version": "0.2.0",
+      "mode": "standard",
+      "elapsed_seconds": 0.8394920780000348,
+      "peak_rss_mb": 264.8984375,
+      "profiled_columns": 64,
+      "backend_status": {
+        "selected": "rust",
+        "native_available": true,
+        "environment_override": "rust",
+        "eligible": [
+          "numpy",
+          "rust"
+        ]
+      },
+      "full_materialization": null
+    }
+  ]
+}
diff --git a/tests/test_benchmark_harness.py b/tests/test_benchmark_harness.py
index 0a5ed97..f2f79a4 100644
--- a/tests/test_benchmark_harness.py
+++ b/tests/test_benchmark_harness.py
@@ -1,5 +1,7 @@
 import json
+import math
 from pathlib import Path
+import statistics
 import subprocess
 import sys
 
@@ -40,3 +42,99 @@ def test_scale_benchmark_harness_emits_machine_readable_result(tmp_path):
     assert measurement["peak_rss_mb"] > 0
     assert measurement["result"]["shape"] == {"rows": 200, "columns": 5}
     assert json.loads(completed.stdout)["workload"] == payload["workload"]
+
+
+def test_release_delta_benchmark_evidence_is_self_consistent():
+    evidence = Path(
+        "benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json"
+    )
+    payload = json.loads(evidence.read_text(encoding="utf-8"))
+
+    assert payload["benchmark_schema_version"] == 1
+    assert payload["comparison"] == "FrameVitals 0.2.0 vs 0.1.0"
+    assert payload["old_ref"] == (
+        "v0.1.0@3da1432168fbfcb3dbe99fcfb6f6200f5e63214b"
+    )
+    assert payload["new_ref"] == (
+        "develop/august@05b11e594995a3833ec08f5b4a8a145197bf4cab"
+    )
+
+    dataset = payload["dataset"]
+    assert dataset["rows"] == 10_000
+    assert dataset["columns"] == 64
+    assert dataset["cells"] == 640_000
+    assert dataset["format"] == "csv"
+
+    methodology = payload["methodology"]
+    assert methodology["warmups_per_version_mode"] == 1
+    assert methodology["measured_repetitions_per_version_mode"] == 3
+    assert methodology["measurement_order"] == "ABBAAB"
+    assert methodology["scope"] == (
+        "all 10,000 rows and all 64 columns in both releases"
+    )
+
+    measurements = payload["measurements"]
+    assert len(measurements) == 12
+    assert {(item["version"], item["mode"]) for item in measurements} == {
+        ("0.1.0", "quick"),
+        ("0.2.0", "quick"),
+        ("0.1.0", "standard"),
+        ("0.2.0", "standard"),
+    }
+
+    for mode in ("quick", "standard"):
+        mode_result = payload["modes"][mode]
+        old = mode_result["0.1.0"]
+        new = mode_result["0.2.0"]
+
+        for version_result in (old, new):
+            assert version_result["profiled_columns"] == 64
+            assert len(version_result["wall_seconds"]) == 3
+            assert len(version_result["peak_rss_mb"]) == 3
+            assert math.isclose(
+                version_result["median_wall_seconds"],
+                statistics.median(version_result["wall_seconds"]),
+                rel_tol=0.0,
+                abs_tol=1e-12,
+            )
+            assert math.isclose(
+                version_result["median_peak_rss_mb"],
+                statistics.median(version_result["peak_rss_mb"]),
+                rel_tol=0.0,
+                abs_tol=1e-12,
+            )
+
+        assert old["backend"]["selected"] == "legacy-python"
+        assert old["backend"]["native_available"] is False
+        assert new["backend"]["selected"] == "rust"
+        assert new["backend"]["native_available"] is True
+
+        expected_speedup = (
+            old["median_wall_seconds"] / new["median_wall_seconds"]
+        )
+        expected_wall_reduction = (
+            1.0
+            - new["median_wall_seconds"] / old["median_wall_seconds"]
+        ) * 100.0
+        expected_rss_reduction = (
+            1.0
+            - new["median_peak_rss_mb"] / old["median_peak_rss_mb"]
+        ) * 100.0
+
+        assert math.isclose(
+            mode_result["speedup_x"],
+            expected_speedup,
+            rel_tol=1e-12,
+        )
+        assert math.isclose(
+            mode_result["wall_time_reduction_percent"],
+            expected_wall_reduction,
+            rel_tol=1e-12,
+        )
+        assert math.isclose(
+            mode_result["peak_rss_reduction_percent"],
+            expected_rss_reduction,
+            rel_tol=1e-12,
+        )
+
+        assert new["median_wall_seconds"] < old["median_wall_seconds"]

From 13e16791f6f3e1fee74ade79c68cbeaf60cee9dc Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 14:13:50 +0530
Subject: [PATCH 434/439] test: keep release branding check optional-safe

---
 tests/test_release_hardening.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/tests/test_release_hardening.py b/tests/test_release_hardening.py
index f9166f5..1e23e67 100644
--- a/tests/test_release_hardening.py
+++ b/tests/test_release_hardening.py
@@ -1,4 +1,4 @@
-import inspect
+from pathlib import Path
 import warnings
 
 import numpy as np
@@ -12,9 +12,11 @@
 def test_materialized_duplicate_estimate_uses_stratified_jitter(monkeypatch):
     frame = pd.DataFrame({"value": [1, 1, 2, 2, 3, 3, 4, 4]})
     seen = {}
+
     def positions(rows, target_rows):
         seen["args"] = (rows, target_rows)
         return np.array([0, 2, 4, 6], dtype=np.int64)
+
     monkeypatch.setattr(profiler, "MAX_EXACT_DUPLICATE_CELLS", 1)
     monkeypatch.setattr(profiler, "DUPLICATE_SAMPLE_ROWS", 4)
     monkeypatch.setattr(profiler, "_deterministic_stratified_positions", positions)
@@ -26,7 +28,12 @@ def positions(rows, target_rows):
 
 def test_pandas_profile_excludes_infinities_from_finite_moments_without_hiding_missingness(monkeypatch):
     monkeypatch.setenv("FRAMEVITALS_BACKEND", "numpy")
-    frame = pd.DataFrame({"x": [1.0, 2.0, np.inf, -np.inf, np.nan], "y": [2.0, 4.0, 8.0, 16.0, 32.0]})
+    frame = pd.DataFrame(
+        {
+            "x": [1.0, 2.0, np.inf, -np.inf, np.nan],
+            "y": [2.0, 4.0, 8.0, 16.0, 32.0],
+        }
+    )
     with warnings.catch_warnings():
         warnings.simplefilter("error", RuntimeWarning)
         profile = build_profile(frame)
@@ -41,8 +48,13 @@ def test_pandas_profile_excludes_infinities_from_finite_moments_without_hiding_m
 
 
 def test_pdf_report_builder_contains_only_framevitals_branding():
-    import framevitals.pdf_report_builder as report_builder
-    source = inspect.getsource(report_builder)
+    source_path = (
+        Path(__file__).resolve().parents[1]
+        / "src"
+        / "framevitals"
+        / "pdf_report_builder.py"
+    )
+    source = source_path.read_text(encoding="utf-8")
     assert "DataLens" not in source
     assert "DATALENS" not in source
     assert "FrameVitals Dataset Report" in source

From c2940280bc32c2c6228775fc13462cea7dbf9490 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 15:08:31 +0530
Subject: [PATCH 435/439] bench: record same-dataset 0.2.0 accuracy parity

---
 ...lease_0.2.0_vs_0.1.0_accuracy_10k_x64.json | 104 ++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100644 benchmarks/results/release_0.2.0_vs_0.1.0_accuracy_10k_x64.json

diff --git a/benchmarks/results/release_0.2.0_vs_0.1.0_accuracy_10k_x64.json b/benchmarks/results/release_0.2.0_vs_0.1.0_accuracy_10k_x64.json
new file mode 100644
index 0000000..c4bd357
--- /dev/null
+++ b/benchmarks/results/release_0.2.0_vs_0.1.0_accuracy_10k_x64.json
@@ -0,0 +1,104 @@
+{
+  "benchmark_schema_version": 1,
+  "comparison": "FrameVitals 0.2.0 vs 0.1.0 same-dataset statistical accuracy",
+  "dataset": {
+    "bytes": 2811188,
+    "cells": 640000,
+    "columns": 64,
+    "format": "csv",
+    "generator": "((row * (col + 3) + col * 17) % 2001 - 1000).astype(int16)",
+    "rows": 10000,
+    "same_as_release_performance_run": 32010158292
+  },
+  "delta_new_minus_old": {
+    "max_exact_fact_absolute_error": 0.0,
+    "max_mean_absolute_error": 0.0,
+    "max_quantile_absolute_error": 4.196000000000026,
+    "max_shape_absolute_error": 0.0,
+    "max_std_absolute_error": 0.0,
+    "pearson_absolute_error": 0.0
+  },
+  "evidence_runs": {
+    "accuracy_full_legacy_and_initial_native_run": 32014979365,
+    "native_shape_corrected_run": 32015665811,
+    "same_dataset_performance_run": 32010158292
+  },
+  "methodology": {
+    "dependency_stack": {
+      "numpy": "1.26.0",
+      "pandas": "2.2.0",
+      "pyarrow": "25.0.0",
+      "scikit-learn": "1.5.0",
+      "scipy": "1.13.0",
+      "statsmodels": "0.14.0"
+    },
+    "notes": [
+      "0.1.0 output came from accuracy run 32014979365.",
+      "0.2.0 shape fields were reprobed in run 32015665811 because the initial extractor looked only in deep_statistics_v2 instead of the profile's full-stream M3/M4 fields.",
+      "The product engine source under test is develop/august@13e16791; temporary benchmark-only files did not modify FrameVitals engine code."
+    ],
+    "public_api": "framevitals.analyze(path, mode='standard', artifacts=False)",
+    "python": "3.11",
+    "runner": "GitHub Actions ubuntu-latest",
+    "scope": "five representative numeric columns plus Pearson correlation; full 10,000 x 64 Standard analysis in both releases",
+    "truth_source": "serialized CSV recomputed independently with pandas/scipy"
+  },
+  "new_ref": "develop/august@13e16791f6f3e1fee74ade79c68cbeaf60cee9dc",
+  "old_ref": "v0.1.0@3da1432168fbfcb3dbe99fcfb6f6200f5e63214b",
+  "quantile_error_context": {
+    "max_absolute_error_units": 4.196000000000026,
+    "max_error_percent_of_column_range": 0.21001001001001132,
+    "mean_absolute_error_units": 1.589200000000014,
+    "mean_error_percent_of_column_range": 0.07949557557557627,
+    "native_quantile_relative_accuracy_setting": 0.01,
+    "tracked_quantile_values": 15
+  },
+  "tracked_columns": ["c000", "c001", "c007", "c031", "c063"],
+  "truth_method": "independent pandas/scipy calculation from the serialized CSV",
+  "versions": {
+    "0.1.0": {
+      "backend": {"native_available": false, "selected": "legacy-python"},
+      "column_absolute_errors": {
+        "c000": {"count": 0.0, "kurtosis": 2.565117664943273e-07, "max": 0.0, "mean": 0.0004999999999999449, "median": 0.0, "min": 0.0, "missing": 0.0, "q25": 0.0, "q75": 0.0, "skewness": 9.411780260499463e-08, "std": 0.0004745087312585383},
+        "c001": {"count": 0.0, "kurtosis": 3.374129284861027e-07, "max": 0.0, "mean": 0.00040000000000001146, "median": 0.0, "min": 0.0, "missing": 0.0, "q25": 0.0, "q75": 0.0, "skewness": 3.7097656380637127e-07, "std": 0.0003787506919934458},
+        "c007": {"count": 0.0, "kurtosis": 4.12596387722175e-07, "max": 0.0, "mean": 0.0005000000000000004, "median": 0.0, "min": 0.0, "missing": 0.0, "q25": 0.0, "q75": 0.0, "skewness": 3.5892954760251804e-07, "std": 0.00033771722223718825},
+        "c031": {"count": 0.0, "kurtosis": 4.915813716088735e-07, "max": 0.0, "mean": 0.0005000000000000004, "median": 0.0, "min": 0.0, "missing": 0.0, "q25": 0.0, "q75": 0.0, "skewness": 2.2877938012566199e-07, "std": 5.640241624860209e-07},
+        "c063": {"count": 0.0, "kurtosis": 1.1517126541349398e-07, "max": 0.0, "mean": 0.0, "median": 0.0, "min": 0.0, "missing": 0.0, "q25": 0.0, "q75": 0.0, "skewness": 4.42799937819564e-07, "std": 6.505523697342142e-05}
+      },
+      "summary": {
+        "max_exact_fact_absolute_error": 0.0,
+        "max_mean_absolute_error": 0.0005000000000000004,
+        "max_quantile_absolute_error": 0.0,
+        "max_shape_absolute_error": 4.915813716088735e-07,
+        "max_std_absolute_error": 0.0004745087312585383,
+        "mean_mean_absolute_error": 0.00037999999999999146,
+        "mean_quantile_absolute_error": 0.0,
+        "mean_shape_absolute_error": 3.1088769516840826e-07,
+        "pearson_absolute_error": 0.00014689456555781744,
+        "shape_values_unavailable": 0
+      }
+    },
+    "0.2.0": {
+      "backend": {"eligible": ["numpy", "rust"], "environment_override": "rust", "native_available": true, "selected": "rust"},
+      "column_absolute_errors": {
+        "c000": {"count": 0.0, "kurtosis": 2.565117664943273e-07, "max": 0.0, "mean": 0.0004999999999999449, "median": 0.010000000000000009, "min": 0.0, "missing": 0.0, "q25": 4.196000000000026, "q75": 0.053999999999973625, "skewness": 9.411780260499463e-08, "std": 0.0004745087312585383},
+        "c001": {"count": 0.0, "kurtosis": 3.374129284861027e-07, "max": 0.0, "mean": 0.00040000000000001146, "median": 0.0, "min": 0.0, "missing": 0.0, "q25": 2.1960000000000264, "q75": 2.1960000000000264, "skewness": 3.7097656380637127e-07, "std": 0.0003787506919934458},
+        "c007": {"count": 0.0, "kurtosis": 4.12596387722175e-07, "max": 0.0, "mean": 0.0005000000000000004, "median": 0.5, "min": 0.0, "missing": 0.0, "q25": 1.4460000000000264, "q75": 2.4460000000000264, "skewness": 3.5892954760251804e-07, "std": 0.00033771722223718825},
+        "c031": {"count": 0.0, "kurtosis": 4.915813716088735e-07, "max": 0.0, "mean": 0.0005000000000000004, "median": 0.5, "min": 0.0, "missing": 0.0, "q25": 1.4460000000000264, "q75": 2.4460000000000264, "skewness": 2.2877938012566199e-07, "std": 5.640241624860209e-07},
+        "c063": {"count": 0.0, "kurtosis": 1.1517126541349398e-07, "max": 0.0, "mean": 0.0, "median": 0.010000000000000009, "min": 0.0, "missing": 0.0, "q25": 4.196000000000026, "q75": 2.1960000000000264, "skewness": 4.42799937819564e-07, "std": 6.505523697342142e-05}
+      },
+      "summary": {
+        "max_exact_fact_absolute_error": 0.0,
+        "max_mean_absolute_error": 0.0005000000000000004,
+        "max_quantile_absolute_error": 4.196000000000026,
+        "max_shape_absolute_error": 4.915813716088735e-07,
+        "max_std_absolute_error": 0.0004745087312585383,
+        "mean_mean_absolute_error": 0.00037999999999999146,
+        "mean_quantile_absolute_error": 1.589200000000014,
+        "mean_shape_absolute_error": 3.1088769516840826e-07,
+        "pearson_absolute_error": 0.00014689456555781744,
+        "shape_values_unavailable": 0
+      }
+    }
+  }
+}

From 235d5c8e96067106e73f02acab708d0dc2b0f6fe Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 15:08:57 +0530
Subject: [PATCH 436/439] docs: pair 0.2.0 speedup with same-dataset accuracy
 evidence

---
 CHANGELOG.md | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 037f0b4..d5fc975 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -73,7 +73,20 @@ For an end-to-end release comparison, `v0.1.0` (`3da1432`) and the 0.2.0 release
 
 The measured Quick ranges were 1.620–1.648 s for 0.1.0 and 0.686–0.692 s for 0.2.0. Standard ranges were 140.709–141.130 s for 0.1.0 and 0.823–0.841 s for 0.2.0. The large Standard improvement is a release-level architectural result: 0.2.0 replaces 0.1.0's fully materialized, unbounded heavy-statistics path with bounded/adaptive execution and native streaming. It should not be interpreted as a same-algorithm microbenchmark.
 
-The machine-readable evidence is committed as `benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json` from GitHub Actions run `32010158292`.
+The exact same serialized CSV was then graded against independent pandas/SciPy ground truth. The full Standard pipeline still processed all 10,000 rows and all 64 columns in both releases; the accuracy comparison tracks five representative numeric columns (`c000`, `c001`, `c007`, `c031`, `c063`) plus Pearson correlation for `c000`↔`c001`.
+
+| Tracked metric | 0.1.0 max absolute error | 0.2.0 max absolute error |
+| --- | ---: | ---: |
+| Count / missing / min / max | **0** | **0** |
+| Mean | 0.0005000 | 0.0005000 |
+| Standard deviation | 0.0004745 | 0.0004745 |
+| Skewness / excess kurtosis | 4.92×10⁻⁷ | 4.92×10⁻⁷ |
+| Pearson correlation | 0.0001469 | 0.0001469 |
+| q25 / median / q75 | **0** | 4.196 max; 1.589 mean |
+
+For every tested non-quantile metric, 0.2.0 had **exactly the same measured absolute error as 0.1.0 at the published output precision**. The quantile difference is intentional: 0.2.0 uses a full-stream native log-quantile sketch configured for 1% relative accuracy instead of materializing exact profile quartiles. Across the 15 tracked q25/median/q75 values, the maximum absolute error was 4.196 units and the mean was 1.589 units; normalized by each approximately 2,000-unit observed column range, that is **0.210% max** and **0.0795% mean**. Thus the measured 168.06× Standard speedup and 90.3% RSS reduction preserved the tested exact facts, moments, shape statistics, and correlation fidelity while trading exact profile quartiles for explicitly approximate streaming estimates.
+
+Timing/RSS evidence is committed as `benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json` from GitHub Actions run `32010158292`. Same-dataset accuracy evidence is committed as `benchmarks/results/release_0.2.0_vs_0.1.0_accuracy_10k_x64.json`; the legacy/current accuracy run was `32014979365`, with a current-only shape-field extractor correction verified in run `32015665811`.
 
 A separate 100,000 × 64 (6.4M-cell) stress run completed all Quick repetitions at a 5.097 s vs 1.447 s median (**3.52× faster**) and 406.1 MB vs 282.7 MB median peak RSS (**30.4% lower**) for 0.1.0 vs 0.2.0. The repeated Standard comparison intentionally has no formal speedup claim: the 30-minute workflow limit expired because completed 0.1.0 Standard passes took about 640–649 s and roughly 12.4 GB peak RSS each, while observed 0.2.0 Standard passes completed in about 1.58–2.06 s at roughly 303–308 MB. That incomplete stress evidence is GitHub Actions run `32008727705`.
 

From 1eff6006f7e736f64294c4a058aa689fec723452 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 15:09:21 +0530
Subject: [PATCH 437/439] test: validate same-dataset release accuracy evidence

---
 tests/test_benchmark_harness.py | 93 +++++++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+)

diff --git a/tests/test_benchmark_harness.py b/tests/test_benchmark_harness.py
index f2f79a4..1c1a832 100644
--- a/tests/test_benchmark_harness.py
+++ b/tests/test_benchmark_harness.py
@@ -138,3 +138,96 @@ def test_release_delta_benchmark_evidence_is_self_consistent():
         )
 
         assert new["median_wall_seconds"] < old["median_wall_seconds"]
+
+
+def test_release_accuracy_evidence_matches_same_performance_dataset_and_contract():
+    evidence = Path(
+        "benchmarks/results/release_0.2.0_vs_0.1.0_accuracy_10k_x64.json"
+    )
+    payload = json.loads(evidence.read_text(encoding="utf-8"))
+
+    assert payload["benchmark_schema_version"] == 1
+    assert payload["comparison"] == (
+        "FrameVitals 0.2.0 vs 0.1.0 same-dataset statistical accuracy"
+    )
+    assert payload["old_ref"] == (
+        "v0.1.0@3da1432168fbfcb3dbe99fcfb6f6200f5e63214b"
+    )
+
+    dataset = payload["dataset"]
+    assert dataset == {
+        "bytes": 2_811_188,
+        "cells": 640_000,
+        "columns": 64,
+        "format": "csv",
+        "generator": "((row * (col + 3) + col * 17) % 2001 - 1000).astype(int16)",
+        "rows": 10_000,
+        "same_as_release_performance_run": 32010158292,
+    }
+    assert payload["tracked_columns"] == [
+        "c000",
+        "c001",
+        "c007",
+        "c031",
+        "c063",
+    ]
+    assert payload["evidence_runs"] == {
+        "accuracy_full_legacy_and_initial_native_run": 32014979365,
+        "native_shape_corrected_run": 32015665811,
+        "same_dataset_performance_run": 32010158292,
+    }
+
+    old = payload["versions"]["0.1.0"]
+    new = payload["versions"]["0.2.0"]
+    assert old["backend"] == {
+        "native_available": False,
+        "selected": "legacy-python",
+    }
+    assert new["backend"]["selected"] == "rust"
+    assert new["backend"]["native_available"] is True
+
+    old_summary = old["summary"]
+    new_summary = new["summary"]
+    unchanged_metrics = [
+        "max_exact_fact_absolute_error",
+        "max_mean_absolute_error",
+        "max_std_absolute_error",
+        "max_shape_absolute_error",
+        "pearson_absolute_error",
+    ]
+    for metric in unchanged_metrics:
+        assert math.isclose(
+            new_summary[metric],
+            old_summary[metric],
+            rel_tol=0.0,
+            abs_tol=1e-15,
+        )
+        assert math.isclose(
+            payload["delta_new_minus_old"][metric],
+            0.0,
+            rel_tol=0.0,
+            abs_tol=1e-15,
+        )
+
+    assert old_summary["max_exact_fact_absolute_error"] == 0.0
+    assert new_summary["max_exact_fact_absolute_error"] == 0.0
+    assert old_summary["shape_values_unavailable"] == 0
+    assert new_summary["shape_values_unavailable"] == 0
+
+    quantiles = payload["quantile_error_context"]
+    assert quantiles["tracked_quantile_values"] == 15
+    assert quantiles["native_quantile_relative_accuracy_setting"] == 0.01
+    assert math.isclose(
+        new_summary["max_quantile_absolute_error"],
+        quantiles["max_absolute_error_units"],
+        rel_tol=0.0,
+        abs_tol=1e-12,
+    )
+    assert math.isclose(
+        new_summary["mean_quantile_absolute_error"],
+        quantiles["mean_absolute_error_units"],
+        rel_tol=0.0,
+        abs_tol=1e-12,
+    )
+    assert quantiles["max_error_percent_of_column_range"] < 0.211
+    assert quantiles["mean_error_percent_of_column_range"] < 0.080

From e47aa24de13ba353a8194784b45f604c2658dd26 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Mon, 17 Aug 2026 18:44:00 +0530
Subject: [PATCH 438/439] docs: update README for FrameVitals 0.2.0

---
 README.md | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 6c300bd..bfaeb1c 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
 [![GitHub stars](https://img.shields.io/github/stars/parthdongre/FrameVitals?style=social)](https://github.com/parthdongre/FrameVitals)
 
-[Install](#installation) · [Quick start](#quick-start) · [Quality gates](#quality-gates) · [Sources](#source-aware-execution) · [CLI](#command-line-interface) · [Roadmap](#roadmap) · [Contributing](#contributing)
+[Install](#installation) · [Quick start](#quick-start) · [Performance](#020-performance-and-fidelity) · [Quality gates](#quality-gates) · [Sources](#source-aware-execution) · [CLI](#command-line-interface) · [Roadmap](#roadmap) · [Contributing](#contributing)
 
 </div>
 
@@ -80,12 +80,14 @@ FrameVitals is **package-first**. Product logic lives under `src/framevitals/`;
 
 ## Installation
 
-FrameVitals supports **Python 3.11, 3.12, and 3.13**. The current public release is **0.1.0 (alpha)**.
+FrameVitals supports **Python 3.11, 3.12, and 3.13**. The current public release is **0.2.0**.
 
 ```bash
 pip install framevitals
 ```
 
+FrameVitals 0.2.0 publishes native ABI3 wheels for supported Linux, macOS, and Windows targets while retaining a portable pure-Python fallback wheel and source distribution. When a compatible native wheel is available, FrameVitals can route supported hot paths through the Rust backend automatically; otherwise the Python/NumPy fallback remains available.
+
 Optional capabilities are split into extras so the base data-health engine stays focused:
 
 ```bash
@@ -296,7 +298,7 @@ FrameVitals does not import or execute installed check plugins automatically.
 The repository ships a reusable composite action at `action.yml`:
 
 ```yaml
-- uses: parthdongre/FrameVitals@main
+- uses: parthdongre/FrameVitals@v0.2.0
   id: framevitals
   with:
     current: data/production.parquet
@@ -446,6 +448,21 @@ A source being streamable does **not** mean every operation is approximate. Fram
 - exact contracts and arbitrary custom checks remain full-data operations;
 - every source-aware result should disclose whether data was streamed, sampled, estimated or fully materialized.
 
+## 0.2.0 performance and fidelity
+
+FrameVitals does **not** claim a universal speedup. Performance depends on source shape, mode, backend, storage and statistical fidelity. The release comparison below uses the same deterministic physical 10,000 × 64 CSV (640,000 cells), the same Python/dependency stack, one warm-up and three interleaved measured runs per release/mode. FrameVitals 0.2.0 used the native Rust backend and both releases analyzed all rows and all 64 columns.
+
+| Mode | 0.1.0 median wall | 0.2.0 median wall | Speedup | 0.1.0 peak RSS | 0.2.0 peak RSS |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| Quick | 1.644 s | 0.691 s | **2.38×** | 234.7 MB | 239.5 MB |
+| Standard | 141.084 s | 0.839 s | **168.06×** | 2736.7 MB | 264.9 MB |
+
+The large Standard improvement is an end-to-end architectural result, not a same-algorithm Rust-vs-Python microbenchmark. FrameVitals 0.2.0 replaces the old fully materialized, unbounded heavy-statistics path with source-aware bounded/adaptive execution, exact-once reuse and native streaming kernels.
+
+Accuracy was independently graded on five representative columns plus one Pearson pair from that exact benchmark dataset. Tested count/missing/min/max facts remained exact, and the measured mean, standard deviation, skewness/kurtosis and Pearson errors were unchanged from 0.1.0 at the recorded output precision. The deliberate fidelity trade-off is native streaming quantiles: across tested q25/median/q75 values, 0.2.0 measured about **0.08% mean** and **0.21% max** absolute error normalized to the observed column range.
+
+The repository also contains physical scale validation through **5 billion logical cells** and machine-readable benchmark evidence under `benchmarks/results/`. See [CHANGELOG.md](CHANGELOG.md) for methodology and caveats.
+
 ## Filesystem artifacts are opt-in
 
 Calling the Python API does not need to scatter reports and cleaned files around the working directory.
@@ -607,7 +624,7 @@ Typical local endpoints:
 ├── src/framevitals/          # canonical installable Python package
 ├── tests/                    # automated test suite
 ├── benchmarks/               # scale/performance harnesses
-├── rust/                     # optional native acceleration core
+├── rust/                     # native acceleration core
 ├── frontend/                 # optional React + TypeScript dashboard
 ├── templates/                # Flask report pages
 ├── static/                   # web/report assets
@@ -656,23 +673,23 @@ FrameVitals is moving toward a dependable data-health quality gate rather than a
      analyze · focused diagnostics · drift · package/CLI baseline
 
 0.2  SOURCE-AWARE QUALITY GATES
-     bounded streaming · contracts · validate · gate · snapshots · provenance
+     bounded streaming · native execution · contracts · validate · gate · snapshots · provenance
 
 0.3  STABLE RESULTS + PERFORMANCE
-     result-schema hardening · regression budgets · lighter installs · richer history
+     result-schema hardening · planner/cost fidelity · regression budgets · richer history
 
 0.4  EXTENSIBILITY + INTEROPERABILITY
-     custom checks · entry-point plugins · source protocols · integrations
+     ecosystem checks · source protocols · integrations · advanced streaming statistics
 
 1.0  STABLE DATA-HEALTH API
      dependable analyze → compare → validate → gate → monitor workflow
 ```
 
-Several post-0.1 capabilities are already under active development on repository development branches. Before `1.0`, naming, thresholds, result schemas and extension points may still evolve.
+FrameVitals 0.2.0 delivers the source-aware quality-gate architecture. Before `1.0`, naming, thresholds, result schemas and extension points may still evolve.
 
 ## Project status
 
-FrameVitals `0.1.x` is **alpha software**. The project already has a package API, CLI, contracts, gates, snapshots, source-aware execution, tests and release tooling, while the `0.x` series deliberately leaves room to refine schemas, thresholds, dependency boundaries and extension points before stability guarantees begin.
+FrameVitals `0.2.x` is **pre-1.0 software**. The project now has a package API, CLI, native/portable execution paths, contracts, gates, snapshots, source-aware streaming, exact-once statistical reuse, reproducible performance evidence, broad CI coverage and release tooling, while the `0.x` series deliberately leaves room to refine schemas, thresholds, dependency boundaries and extension points before stability guarantees begin.
 
 Feedback on real datasets, false positives, missing diagnostics, performance, source compatibility and API ergonomics is especially valuable.
 

From 9aaa4a2f790ad88407be285299a9f75d794328d2 Mon Sep 17 00:00:00 2001
From: Parthozy <163404890+parthdongre@users.noreply.github.com>
Date: Tue, 18 Aug 2026 10:46:42 +0530
Subject: [PATCH 439/439] Simplify and tighten README

---
 README.md | 647 +++++++++++-------------------------------------------
 1 file changed, 129 insertions(+), 518 deletions(-)

diff --git a/README.md b/README.md
index bfaeb1c..92f969f 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@
 
 ### Know if your data is healthy, stable, and ML-ready — before your model finds out.
 
-**A source-aware Python toolkit for data health, drift, contracts, quality gates, anomaly analysis, and ML-readiness diagnostics on tabular data.**
+**FrameVitals is a source-aware Python toolkit for data health, drift detection, anomaly analysis, data contracts, quality gates, and ML-readiness diagnostics on tabular data.**
 
 [![Tests](https://github.com/parthdongre/FrameVitals/actions/workflows/test.yml/badge.svg)](https://github.com/parthdongre/FrameVitals/actions/workflows/test.yml)
 [![PyPI](https://img.shields.io/pypi/v/framevitals.svg)](https://pypi.org/project/framevitals/)
@@ -12,13 +12,15 @@
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
 [![GitHub stars](https://img.shields.io/github/stars/parthdongre/FrameVitals?style=social)](https://github.com/parthdongre/FrameVitals)
 
-[Install](#installation) · [Quick start](#quick-start) · [Performance](#020-performance-and-fidelity) · [Quality gates](#quality-gates) · [Sources](#source-aware-execution) · [CLI](#command-line-interface) · [Roadmap](#roadmap) · [Contributing](#contributing)
+[Installation](#installation) · [Quick Start](#quick-start) · [Workflows](#common-workflows) · [CLI](#command-line) · [Docs](docs/) · [Contributing](CONTRIBUTING.md)
 
 </div>
 
 ---
 
-FrameVitals turns a supported tabular source into a **structured health report** that can be inspected, serialized, compared, snapshotted, extended with domain rules, and enforced in CI.
+FrameVitals helps you decide whether a dataset is **healthy enough to trust** before it reaches a model, analytics workflow, dashboard, or production pipeline.
+
+It provides one consistent API for inspecting data quality, ML readiness, anomalies, drift, contracts, validation, snapshots, and CI-friendly quality gates.
 
 ```python
 import framevitals as fv
@@ -30,177 +32,72 @@ validation = fv.validate(current, contract)
 gate = fv.gate(current, reference=reference, contract=contract)
 ```
 
-The goal is simple: **catch bad data before it becomes a bad model, a broken dashboard, or a production incident.**
-
-```text
-DataFrame / Arrow / files / DuckDB relation
-                    │
-                    ▼
-              DatasetSource
-                    │
-          ┌─────────┴─────────┐
-          ▼                   ▼
-     bounded stream       exact/full path
-          │                   │
-          └─────────┬─────────┘
-                    ▼
-                 ANALYZE
-                    │
-      ┌─────────────┼─────────────┐
-      ▼             ▼             ▼
-   health        contract      snapshot
-                    │             │
-            ┌───────┘             ▼
-            ▼                  history
-         validate                 │
-            │                     │
-reference ─► compare              │
-            │                     │
-custom ─────┤                     │
-checks      ▼                     │
-           GATE ◄─────────────────┘
-      PASS / WARN / FAIL
-```
-
-## Why FrameVitals?
-
-| Question | FrameVitals |
-| --- | --- |
-| Is this dataset structurally healthy? | Missingness, duplicates, cardinality, schema and quality diagnostics |
-| Is it ready for modelling? | ML-readiness scoring, target-aware checks and model diagnostics |
-| Are there suspicious rows or features? | Statistical diagnostics, anomaly detection, leakage and multicollinearity checks |
-| Has production data changed? | Numeric, categorical and schema drift diagnostics |
-| Can I enforce expectations? | Versioned contracts, exact validation, custom checks and one quality gate |
-| Can I monitor change without storing every raw batch? | Compact snapshots and persistent snapshot history |
-| Will large supported sources be loaded into pandas by default? | Bounded source-aware execution where semantics permit it |
-| Can I see when FrameVitals sampled or materialized data? | Execution provenance is included in source-aware results |
-| Will analysis unexpectedly write files? | No — filesystem artifacts are opt-in |
-
-FrameVitals is **package-first**. Product logic lives under `src/framevitals/`; the CLI, Flask API, React dashboard and reusable GitHub Action wrap the same canonical engines.
+The goal is simple: **catch bad data before it becomes a bad model, broken dashboard, or production incident.**
 
 ## Installation
 
-FrameVitals supports **Python 3.11, 3.12, and 3.13**. The current public release is **0.2.0**.
+Install the package from PyPI:
 
 ```bash
 pip install framevitals
 ```
 
-FrameVitals 0.2.0 publishes native ABI3 wheels for supported Linux, macOS, and Windows targets while retaining a portable pure-Python fallback wheel and source distribution. When a compatible native wheel is available, FrameVitals can route supported hot paths through the Rust backend automatically; otherwise the Python/NumPy fallback remains available.
+FrameVitals supports **Python 3.11, 3.12, and 3.13**.
 
-Optional capabilities are split into extras so the base data-health engine stays focused:
+Optional capabilities are available as extras:
 
 ```bash
-pip install "framevitals[arrow]"   # Parquet + Arrow-backed source interoperability
-pip install "framevitals[duckdb]"  # lazy DuckDB relations + Arrow transport
-pip install "framevitals[excel]"   # XLS/XLSX readers
-pip install "framevitals[plot]"    # Matplotlib/Seaborn charts and report plotting
-pip install "framevitals[ml]"      # XGBoost, LightGBM, PyOD, SHAP
-pip install "framevitals[ai]"      # Ollama-backed AI features
+pip install "framevitals[arrow]"   # Arrow and Parquet interoperability
+pip install "framevitals[duckdb]"  # DuckDB relations
+pip install "framevitals[plot]"    # plotting and report charts
+pip install "framevitals[ml]"      # optional ML diagnostics
+pip install "framevitals[ai]"      # Ollama-backed AI capabilities
 pip install "framevitals[web]"     # Flask web runtime
 pip install "framevitals[all]"     # all optional runtime capabilities
 ```
 
-On a development checkout:
-
-```bash
-pip install -e ".[all,dev]"
-```
-
-`arrow` enables projection-aware Parquet execution, compatible CSV/TSV streaming, native PyArrow table inputs, and Arrow PyCapsule-compatible table producers. `duckdb` adds lazy `DuckDBPyRelation` inputs without putting DuckDB in the base dependency set.
-
-## Quick start
+## Quick Start
 
-### Analyze a DataFrame or file
+Analyze a file directly:
 
 ```python
-import pandas as pd
 import framevitals as fv
 
-customers = pd.read_csv("customers.csv")
-report = fv.analyze(customers)
+report = fv.analyze("customers.csv")
 
 print(report.health["overall_score"])
 print(report.ml_readiness)
 print(report.findings[:3])
 ```
 
-File paths work directly:
+Or pass a pandas DataFrame:
 
 ```python
-report = fv.analyze("customers.parquet", mode="quick")
-```
-
-### Analyze Arrow-native data
-
-With the `arrow` extra installed:
-
-```python
-import pyarrow as pa
+import pandas as pd
 import framevitals as fv
 
-table = pa.table({
-    "age": [21, 34, 48],
-    "income": [30_000, 62_000, 81_000],
-})
-
-report = fv.analyze(table, mode="quick")
-profile = fv.profile(table)
+customers = pd.read_csv("customers.csv")
+report = fv.analyze(customers)
 ```
 
-PyArrow `Table` and `RecordBatch` inputs enter the source-aware batch path instead of being converted to pandas before analysis. Table-like objects implementing the Arrow C Stream / PyCapsule protocol can use the same interoperability boundary when PyArrow is installed.
-
-### Analyze a lazy DuckDB relation
-
-With the `duckdb` extra installed:
-
-```python
-import duckdb
-import framevitals as fv
-
-con = duckdb.connect()
-orders = con.sql("""
-    SELECT *
-    FROM read_parquet('orders/*.parquet')
-""")
-
-profile = fv.profile(orders)
-report = fv.analyze(orders, mode="quick")
-```
+FrameVitals also supports Parquet, PyArrow data, and lazy DuckDB relations when the corresponding optional dependencies are installed.
 
-FrameVitals obtains exact relation shape metadata, pushes requested column projection into the relation, and consumes Arrow record batches. It does not call `.df()` for diagnostics that can stay on the streaming path.
+## Common Workflows
 
 ### Run only the diagnostic you need
 
-```python
-profile = fv.profile("customers.parquet")
-health = fv.health("customers.parquet")
-readiness = fv.ml_readiness("customers.parquet")
-quality = fv.quality("customers.parquet")
-stats = fv.statistics("customers.parquet", mode="quick")
-anomalies = fv.anomalies("customers.parquet", mode="quick")
-relationships = fv.relationships("customers.parquet")
-```
-
-Focused APIs avoid running unrelated pipeline stages. Supported streaming sources use bounded execution where the diagnostic permits it and disclose exact, sampled, estimated, or materialized behavior in result metadata.
-
-### Add a supervised-learning target
+The focused APIs let you inspect one part of a dataset without running the complete analysis pipeline:
 
 ```python
-report = fv.analyze(
-    customers,
-    target="churn",
-    mode="deep",
-)
-
-print(report["model_leaderboard"])
-print(report["explainability"])
+fv.profile(data)
+fv.health(data)
+fv.quality(data)
+fv.ml_readiness(data)
+fv.statistics(data)
+fv.anomalies(data)
+fv.relationships(data)
 ```
 
-Target-aware analysis can surface modelling risks such as leakage, imbalance, redundant features, unstable relationships, and weak baselines.
-
-## Drift and contracts
-
 ### Compare datasets for drift
 
 ```python
@@ -208,12 +105,11 @@ result = fv.compare(reference, current)
 
 print(result.severity)
 print(result["columns"][:3])
-print(result["execution"])
 ```
 
-Numeric drift uses PSI, Kolmogorov-Smirnov statistics and standardized mean shift. Categorical drift uses PSI and chi-square diagnostics. Streaming-capable sources can bound distribution work while preserving exact source-shape metadata.
+Use this to compare training and production data, historical batches, pipeline outputs, or any reference/current pair.
 
-### Infer and validate a contract
+### Infer and validate a data contract
 
 ```python
 contract = fv.infer_contract(reference)
@@ -224,106 +120,58 @@ if result.status == "fail":
         print(finding["message"])
 ```
 
-Contracts are versioned JSON-friendly dictionaries. Current inference can capture required/optional columns, broad data types, nullability, tolerated null rates, finite numeric bounds, low-cardinality allowed values, and uniqueness expectations.
-
-**Validation intentionally remains exact.** Constraints such as uniqueness, allowed values and bounds are not silently downgraded to sampled approximations. For a non-pandas source, exact validation may therefore materialize a complete pandas representation, and the `execution.full_materialization` field reports that decision.
-
-## Quality gates
+Contracts can capture expectations such as schema, data types, nullability, numeric bounds, allowed values, and uniqueness.
 
-`fv.gate(...)` combines the checks you choose into one CI-friendly verdict:
+### Add a quality gate
 
 ```python
 result = fv.gate(
     current,
     reference=reference,
     contract=contract,
-    drift_warn_on="moderate",
-    drift_fail_on="severe",
 )
 
-print(result.status)   # pass / warn / fail
-print(result.passed)   # True unless the gate failed
+print(result.status)  # pass / warn / fail
+print(result.passed)
 ```
 
-You can run only the families you need:
-
-```python
-fv.gate(current, contract=contract)
-fv.gate(current, reference=reference)
-fv.gate(current, reference=reference, contract=contract)
-```
+A gate combines the checks you choose into one verdict that can be used in scripts, pipelines, and CI.
 
-### Domain-specific custom checks
-
-FrameVitals supports exact application-defined invariants without requiring a fork:
+### Add domain-specific checks
 
 ```python
-@fv.check(
-    "positive revenue",
-    severity="error",
-    description="Revenue cannot be negative.",
-)
+@fv.check("positive revenue", severity="error")
 def positive_revenue(df):
     return {
         "passed": bool((df["revenue"] >= 0).all()),
-        "message": "Negative revenue records were found.",
+        "message": "Negative revenue values were found.",
     }
 
-checks = fv.run_checks(current, [positive_revenue])
-gate = fv.gate(current, custom_checks=[positive_revenue])
+result = fv.gate(data, custom_checks=[positive_revenue])
 ```
 
-`fv.run_checks()` returns a dict-compatible `CheckResult`. Arbitrary Python checks run against the complete DataFrame because FrameVitals cannot safely infer whether a user-defined invariant is sampleable. Non-pandas sources therefore report full materialization for this exact path.
-
-### Third-party check plugins
+Custom checks make it possible to enforce application-specific rules without modifying FrameVitals itself.
 
-Installed packages can register checks through Python entry points under `framevitals.checks`:
-
-```toml
-[project.entry-points."framevitals.checks"]
-positive_revenue = "acme_data_checks:positive_revenue"
-```
-
-Discovery is deliberately **opt-in** because loading an entry point executes provider code:
+### Run target-aware analysis
 
 ```python
-checks = fv.discover_checks()
-result = fv.gate(current, custom_checks=checks)
-```
-
-FrameVitals does not import or execute installed check plugins automatically.
-
-### GitHub Actions
-
-The repository ships a reusable composite action at `action.yml`:
-
-```yaml
-- uses: parthdongre/FrameVitals@v0.2.0
-  id: framevitals
-  with:
-    current: data/production.parquet
-    reference: data/training.parquet
-    contract: data/contract.json
-    drift-warn-on: moderate
-    drift-fail-on: severe
-    output: framevitals-gate.json
-
-- name: Show verdict
-  run: echo "FrameVitals status: ${{ steps.framevitals.outputs.status }}"
+report = fv.analyze(
+    data,
+    target="churn",
+    mode="deep",
+)
 ```
 
-For production workflows, pin the action to a released tag or commit rather than a moving branch. The action exposes `status`, `passed`, and `result-path` outputs.
+Target-aware analysis can surface modelling risks such as leakage, imbalance, redundant features, multicollinearity, and weak baseline relationships.
 
-## Snapshots and monitoring history
-
-An `AnalysisResult` can be reduced to a compact versioned snapshot:
+### Create monitoring snapshots
 
 ```python
 report = fv.analyze(current)
 snapshot = report.snapshot("snapshot.json")
 ```
 
-Snapshots retain a fingerprint and compact state such as schema, missingness, health, ML-readiness and finding codes without embedding the full raw dataset.
+Compare compact snapshots later without retaining every raw dataset:
 
 ```python
 previous = fv.load_snapshot("previous.json")
@@ -331,70 +179,9 @@ latest = fv.load_snapshot("snapshot.json")
 change = fv.compare_snapshots(previous, latest)
 ```
 
-For repeated local monitoring, use `SnapshotHistory`:
+## Analysis Modes
 
-```python
-history = fv.SnapshotHistory(".framevitals/history")
-
-history.add(report.snapshot(), label="production")
-latest = history.latest()
-previous = history.previous()
-change = history.compare_latest()
-```
-
-The default `.framevitals/` runtime directory is ignored by Git.
-
-## The public API
-
-FrameVitals keeps workflow entry points at the package root while retaining focused diagnostics for callers that need one answer rather than the complete pipeline.
-
-| API | Purpose |
-| --- | --- |
-| `framevitals.analyze(...)` | Run the configured source-aware analysis workflow |
-| `framevitals.plan(...)` | Preview applicable modules and execution constraints |
-| `framevitals.profile(...)` | Profile structure, missingness and summaries only |
-| `framevitals.roles(...)` | Infer semantic and structural column roles |
-| `framevitals.health(...)` | Calculate data-health diagnostics only |
-| `framevitals.ml_readiness(...)` | Calculate ML-readiness diagnostics only |
-| `framevitals.quality(...)` | Run deterministic data-quality checks only |
-| `framevitals.statistics(...)` | Run bounded deep statistical diagnostics |
-| `framevitals.anomalies(...)` | Run bounded anomaly diagnostics |
-| `framevitals.relationships(...)` | Discover strong numeric relationships |
-| `framevitals.target_analysis(...)` | Run target-aware diagnostics only |
-| `framevitals.compare(...)` | Compare reference/current data for drift |
-| `framevitals.infer_contract(...)` | Infer a reusable versioned data contract |
-| `framevitals.validate(...)` | Validate data against a contract |
-| `framevitals.check(...)` | Define a reusable custom invariant |
-| `framevitals.run_checks(...)` | Run exact custom checks and return `CheckResult` |
-| `framevitals.discover_checks(...)` | Explicitly discover installed check plugins |
-| `framevitals.gate(...)` | Combine contract, custom and drift checks into one verdict |
-| `framevitals.create_snapshot(...)` | Create compact monitoring state from an analysis result |
-| `framevitals.compare_snapshots(...)` | Compare two stored analysis states |
-| `framevitals.SnapshotHistory(...)` | Persist and compare a compact local monitoring timeline |
-
-`framevitals.api` remains a lazy compatibility facade over these canonical engines; it does not maintain a second implementation.
-
-## What FrameVitals checks
-
-| Area | Examples |
-| --- | --- |
-| **Structure** | shape, dtypes, semantic column roles, date/text detection |
-| **Data quality** | missingness, duplicates, constants, cardinality, outliers |
-| **Health scoring** | overall dataset health plus component diagnostics |
-| **ML readiness** | modelling readiness, risky columns, preprocessing recommendations |
-| **Statistics** | distribution checks, normality, correlations, effect-size style diagnostics |
-| **Anomalies** | multivariate and robust outlier detectors, optional ensemble methods |
-| **Target intelligence** | task inference, leakage hints, multicollinearity, feature/model diagnostics |
-| **Drift** | PSI, KS, chi-square, mean shift, new/disappearing categories |
-| **Contracts** | schema, types, nullability, bounds, domains and uniqueness expectations |
-| **Custom invariants** | exact application-specific DataFrame predicates |
-| **Time series** | date-aware diagnostics, stationarity, decomposition and forecast previews |
-| **Text** | text-column profiling, vocabulary and lightweight semantic diagnostics |
-| **Explainability** | model feature importance and SHAP when optional capabilities are installed |
-
-Not every analysis runs on every dataset. FrameVitals uses dataset signals, selected mode, target availability, source capabilities, execution budgets and installed optional dependencies to decide what is useful and safe to execute.
-
-## Analysis modes
+Choose how much work FrameVitals should perform:
 
 ```python
 fv.analyze(data, mode="quick")
@@ -403,314 +190,138 @@ fv.analyze(data, mode="deep")
 fv.analyze(data, mode="research")
 ```
 
-| Mode | Best for |
-| --- | --- |
-| `quick` | Fast structural, quality and ML-readiness checks |
-| `standard` | Everyday analysis with broader diagnostics |
-| `deep` | Target-aware and heavier statistical analysis |
-| `research` | Largest analysis budget for exploratory work |
-
-## Source-aware execution
-
-FrameVitals separates a **dataset source** from the diagnostic that consumes it. This lets supported inputs expose exact metadata, projection and record batches without forcing every public API to begin with a full pandas conversion.
-
-```text
-Pandas DataFrame
-PyArrow Table / RecordBatch
-Arrow C Stream / PyCapsule-compatible table
-CSV / TSV / Parquet
-DuckDB relation
-        │
-        ▼
-   DatasetSource
-        │
- ┌──────┴─────────┐
- ▼                ▼
-stream/bounded   exact/full
- ▼                ▼
-execution provenance
-```
-
-Current source behavior includes:
-
-- **Pandas** — already materialized, canonical in-memory baseline.
-- **Parquet** — Arrow-backed metadata, projection and record batches.
-- **CSV/TSV** — Arrow streaming when compatible and the `arrow` extra is installed; pandas fallback otherwise.
-- **PyArrow Table/RecordBatch** — native in-memory Arrow batch path.
-- **Arrow-compatible table producers** — normalized through the standard Arrow C Stream / PyCapsule boundary when available.
-- **DuckDB relation** — exact count/schema metadata, projection pushed into DuckDB, Arrow batch transport, no `.df()` on streaming-safe diagnostics.
-- **Other supported files** — materialized through the established loader when no streaming adapter exists.
-
-A source being streamable does **not** mean every operation is approximate. FrameVitals chooses execution semantics by diagnostic:
-
-- source shape/schema can remain exact;
-- some statistics, anomaly and drift work can operate on explicit bounded samples;
-- exact contracts and arbitrary custom checks remain full-data operations;
-- every source-aware result should disclose whether data was streamed, sampled, estimated or fully materialized.
-
-## 0.2.0 performance and fidelity
-
-FrameVitals does **not** claim a universal speedup. Performance depends on source shape, mode, backend, storage and statistical fidelity. The release comparison below uses the same deterministic physical 10,000 × 64 CSV (640,000 cells), the same Python/dependency stack, one warm-up and three interleaved measured runs per release/mode. FrameVitals 0.2.0 used the native Rust backend and both releases analyzed all rows and all 64 columns.
-
-| Mode | 0.1.0 median wall | 0.2.0 median wall | Speedup | 0.1.0 peak RSS | 0.2.0 peak RSS |
-| --- | ---: | ---: | ---: | ---: | ---: |
-| Quick | 1.644 s | 0.691 s | **2.38×** | 234.7 MB | 239.5 MB |
-| Standard | 141.084 s | 0.839 s | **168.06×** | 2736.7 MB | 264.9 MB |
-
-The large Standard improvement is an end-to-end architectural result, not a same-algorithm Rust-vs-Python microbenchmark. FrameVitals 0.2.0 replaces the old fully materialized, unbounded heavy-statistics path with source-aware bounded/adaptive execution, exact-once reuse and native streaming kernels.
-
-Accuracy was independently graded on five representative columns plus one Pearson pair from that exact benchmark dataset. Tested count/missing/min/max facts remained exact, and the measured mean, standard deviation, skewness/kurtosis and Pearson errors were unchanged from 0.1.0 at the recorded output precision. The deliberate fidelity trade-off is native streaming quantiles: across tested q25/median/q75 values, 0.2.0 measured about **0.08% mean** and **0.21% max** absolute error normalized to the observed column range.
-
-The repository also contains physical scale validation through **5 billion logical cells** and machine-readable benchmark evidence under `benchmarks/results/`. See [CHANGELOG.md](CHANGELOG.md) for methodology and caveats.
-
-## Filesystem artifacts are opt-in
-
-Calling the Python API does not need to scatter reports and cleaned files around the working directory.
-
-```python
-report = fv.analyze(data)
-assert report["cleaning"]["output_path"] is None
-
-report = fv.analyze(data, artifacts=True)
-print(report["cleaning"]["output_path"])
-```
-
-## Command-line interface
+Use `quick` for fast checks and the deeper modes when you want broader statistical or modelling diagnostics.
 
-FrameVitals ships with a CLI for terminals, scripts and CI workflows.
+## Source-Aware Execution
 
-```bash
-framevitals --help
-framevitals analyze --help
-framevitals compare --help
-framevitals gate --help
-framevitals --version
-```
+FrameVitals is designed to work with more than pandas alone. Supported sources can include DataFrames, files, Arrow-native data, and DuckDB relations.
 
-Analyze and plan:
+Where semantics allow it, large or lazy sources can use bounded or streaming execution instead of being loaded fully into pandas. Operations that require exact results can still materialize the full dataset, and execution metadata reports those decisions.
 
-```bash
-framevitals analyze dataset.csv
-framevitals analyze dataset.csv --mode quick
-framevitals analyze dataset.csv --target churn --mode deep
-framevitals analyze dataset.csv --output report.json
-framevitals plan dataset.csv --mode standard
-```
+## Command Line
 
-Compare, contract and gate:
+The Python package also includes a CLI.
 
-```bash
-framevitals compare train.csv production.csv --fail-on moderate
-framevitals infer-contract training_data.csv --output contract.json
-framevitals validate production_batch.csv --contract contract.json
-framevitals gate production_batch.csv \
-  --reference training_data.csv \
-  --contract contract.json \
-  --output gate.json
-```
-
-CLI exit behavior is intentional:
-
-- `framevitals validate`: `0` for pass/warn by default, `1` for warning when `--fail-on-warn` is enabled, `2` for contract failure.
-- `framevitals compare`: `1` only when `--fail-on` is supplied and the configured drift severity is reached; otherwise `0`.
-- `framevitals gate`: `0` for pass/warn and `1` for fail.
-
-## Result objects
-
-The main workflows return dict-compatible result objects so existing mapping-style code keeps working while application/notebook ergonomics improve:
-
-- `AnalysisResult`
-- `DriftResult`
-- `ValidationResult`
-- `CheckResult`
-- `GateResult`
-- `AnalysisSnapshot`
-
-Example:
-
-```python
-report = fv.analyze(data)
-report.health
-report.findings
-report.column("age")
-report.to_json("report.json")
-report.to_html("report.html")
-report.snapshot("snapshot.json")
-```
-
-The `0.x` series preserves room to harden result schemas before `1.0` stability guarantees.
-
-## Optional capabilities
-
-The default package contains the core data-health engine. Additional integrations are imported only when their feature is requested.
+Analyze a dataset:
 
 ```bash
-pip install "framevitals[arrow]"
+framevitals analyze customers.csv
 ```
 
-Adds Arrow-backed files and in-memory Arrow interoperability.
+Compare two datasets:
 
 ```bash
-pip install "framevitals[duckdb]"
+framevitals compare reference.csv current.csv
 ```
 
-Adds lazy DuckDB relation support plus the Arrow transport required by that adapter.
+Infer a contract:
 
 ```bash
-pip install "framevitals[ml]"
+framevitals infer-contract reference.csv
 ```
 
-Adds heavier model integrations including XGBoost, LightGBM, PyOD and SHAP.
+Create a monitoring snapshot:
 
 ```bash
-pip install "framevitals[ai]"
+framevitals snapshot customers.csv
 ```
 
-Adds Ollama-backed interpretation and question-answering features. AI remains an optional explanation layer; computed diagnostics remain usable without it.
+Inspect dataset execution capabilities:
 
 ```bash
-pip install "framevitals[plot]"
+framevitals inspect customers.csv
 ```
 
-Adds Matplotlib/Seaborn-backed chart rendering and report plotting. Structured diagnostics do not require this extra.
+See all commands and options with:
 
 ```bash
-pip install "framevitals[excel]"
+framevitals --help
 ```
 
-Adds XLS/XLSX reader engines. CSV/TSV/JSON workflows remain available without them.
-
-## Web dashboard
+## CI and GitHub Actions
 
-The repository includes an optional **Flask API + React/TypeScript dashboard** for interactive exploration.
+FrameVitals can sit between your data pipeline and downstream work:
 
-```bash
-pip install -e ".[web]"
-python app.py
+```text
+Data / ETL
+    ↓
+FrameVitals Gate
+    ↓
+PASS / WARN / FAIL
+    ↓
+Training / Analytics / Production
 ```
 
-Then in another terminal:
+The repository includes a reusable GitHub Action:
 
-```bash
-cd frontend
-npm ci
-npm run dev
+```yaml
+- uses: parthdongre/FrameVitals@v0.2.0
+  id: framevitals
+  with:
+    current: data/production.parquet
+    reference: data/training.parquet
+    contract: data/contract.json
+    output: framevitals-gate.json
 ```
 
-The `web` extra contains the server runtime only. Install `plot` for server-side chart/report artifacts and `ai` for the agentic Q&A path. Optional stacks are loaded lazily rather than blocking Flask startup.
+For production workflows, pin the action to a released tag or commit.
 
-Typical local endpoints:
+## Python API
 
-- Flask API: `http://127.0.0.1:5055`
-- React dashboard: `http://127.0.0.1:5173`
+The main workflow entry points are available directly from `framevitals`:
 
-## Design principles
-
-- **Package first** — Python APIs are canonical; interfaces wrap them.
-- **Structured results** — return reusable data, not only screenshots or prose.
-- **Source aware** — inspect and stream supported inputs before choosing materialization.
-- **Bounded execution** — expensive diagnostics operate within explicit scale budgets.
-- **Transparent approximations** — sampled or estimated results disclose provenance.
-- **Exact where correctness requires it** — contracts and arbitrary custom checks are not silently weakened.
-- **Safe defaults** — no unexpected artifact writes or automatic plugin execution.
-- **Small workflow surface** — analyze → compare → validate → gate → monitor stays obvious.
-- **Optional integrations** — Arrow, DuckDB, Excel, plotting, ML, AI and web are independently installable.
-- **Extensible without forks** — domain checks and opt-in entry-point plugins can extend the gate.
-
-## Project layout
-
-```text
-.
-├── src/framevitals/          # canonical installable Python package
-├── tests/                    # automated test suite
-├── benchmarks/               # scale/performance harnesses
-├── rust/                     # native acceleration core
-├── frontend/                 # optional React + TypeScript dashboard
-├── templates/                # Flask report pages
-├── static/                   # web/report assets
-├── examples/                 # focused usage examples
-├── action.yml                # reusable GitHub Actions quality gate
-├── app.py                    # optional Flask API/server
-├── pyproject.toml            # package metadata and dependency groups
-└── .github/workflows/        # CI, interop, package, benchmark and release workflows
-```
-
-New reusable Python code belongs in `src/framevitals/` and should import through the `framevitals.*` namespace.
+```python
+fv.analyze(...)
+fv.plan(...)
+fv.profile(...)
+fv.health(...)
+fv.quality(...)
+fv.ml_readiness(...)
+fv.statistics(...)
+fv.anomalies(...)
+fv.relationships(...)
+fv.compare(...)
+fv.infer_contract(...)
+fv.validate(...)
+fv.check(...)
+fv.run_checks(...)
+fv.gate(...)
+fv.create_snapshot(...)
+fv.compare_snapshots(...)
+```
+
+For detailed API behaviour, configuration, source semantics, performance notes, and advanced usage, see [`docs/`](docs/).
 
 ## Development
 
+Clone the repository and install it in development mode:
+
 ```bash
 git clone https://github.com/parthdongre/FrameVitals.git
 cd FrameVitals
-git switch dev
-
-python -m venv .venv
-source .venv/bin/activate
-python -m pip install --upgrade pip
 pip install -e ".[all,dev]"
-
-pytest
-python -m build
-python -m twine check dist/*
 ```
 
-On Windows PowerShell:
-
-```powershell
-.venv\Scripts\Activate.ps1
-```
-
-CI validates the core package across Python 3.11–3.13, Arrow streaming paths, Arrow/DuckDB interoperability, optional dependency boundaries, the reusable Gate Action, the Rust workspace/native bridge, React build, wheel contents, distribution metadata, release-version consistency, and clean-wheel installation. A separate benchmark workflow records reproducible time and peak-RSS measurements.
-
-Development is integrated through `dev`; `main` is kept release-ready.
+Run the test suite:
 
-## Roadmap
-
-FrameVitals is moving toward a dependable data-health quality gate rather than an ever-growing collection of unrelated analytics modules.
-
-```text
-0.1  FOUNDATION
-     analyze · focused diagnostics · drift · package/CLI baseline
-
-0.2  SOURCE-AWARE QUALITY GATES
-     bounded streaming · native execution · contracts · validate · gate · snapshots · provenance
-
-0.3  STABLE RESULTS + PERFORMANCE
-     result-schema hardening · planner/cost fidelity · regression budgets · richer history
-
-0.4  EXTENSIBILITY + INTEROPERABILITY
-     ecosystem checks · source protocols · integrations · advanced streaming statistics
-
-1.0  STABLE DATA-HEALTH API
-     dependable analyze → compare → validate → gate → monitor workflow
+```bash
+pytest
 ```
 
-FrameVitals 0.2.0 delivers the source-aware quality-gate architecture. Before `1.0`, naming, thresholds, result schemas and extension points may still evolve.
-
-## Project status
-
-FrameVitals `0.2.x` is **pre-1.0 software**. The project now has a package API, CLI, native/portable execution paths, contracts, gates, snapshots, source-aware streaming, exact-once statistical reuse, reproducible performance evidence, broad CI coverage and release tooling, while the `0.x` series deliberately leaves room to refine schemas, thresholds, dependency boundaries and extension points before stability guarantees begin.
-
-Feedback on real datasets, false positives, missing diagnostics, performance, source compatibility and API ergonomics is especially valuable.
-
 ## Contributing
 
-Contributions are welcome. A good contribution is focused, tested, and improves diagnostic reliability, public workflow clarity, source compatibility, execution transparency or extension ergonomics.
-
-Start with [CONTRIBUTING.md](CONTRIBUTING.md), and read the [Code of Conduct](CODE_OF_CONDUCT.md) and [Security Policy](SECURITY.md).
-
-## Releases
-
-Releases are built and validated in GitHub Actions and published through PyPI Trusted Publishing. See [RELEASING.md](RELEASING.md) and [CHANGELOG.md](CHANGELOG.md).
+Contributions are welcome, including bug fixes, diagnostics, tests, documentation, integrations, and performance improvements.
 
-## License
+Please read [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request.
 
-FrameVitals is open source under the [MIT License](LICENSE).
+## Documentation
 
----
+Detailed documentation lives in [`docs/`](docs/).
 
-<div align="center">
+- [Changelog](CHANGELOG.md)
+- [Contributing Guide](CONTRIBUTING.md)
+- [Issue Tracker](https://github.com/parthdongre/FrameVitals/issues)
 
-**If FrameVitals is useful to you, consider starring the repository — it helps the project grow.**
+## License
 
-</div>
+FrameVitals is released under the [MIT License](LICENSE).